public async Task LoadCacheFromStorage()
{
_logger.LogInformation("LoadCacheFromStorage: Starting to LoadCacheFromStorage");
try
{
var expiration = await _cache.GetOrAddAsync($"{_cacheKeyPrefix}expiration", GetAbsoluteExpiration);
var absoluteExpirationInSeconds = DateTimeOffset.Now.AddSeconds(expiration);
var json = await _azureStorageBlogCacheService.RetrieveJsonFromCache();
var agencies = System.Text.Json.JsonSerializer.Deserialize <IList <AgencyFranchiseMap> >(json);
if (agencies.Any())
{
_logger.LogInformation($"LoadCacheFromStorage: {agencies.Count} agencies were loaded from storage.");
_cache.Add($"{_cacheKeyPrefix}agencies", agencies, absoluteExpirationInSeconds);
}
else
{
_logger.LogWarning("LoadCacheFromStorage: WARNING: No agencies were loaded from storage.");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "LoadCacheFromStorage: ERROR: Exception thrown when trying to LoadCacheFromStorage.");
}
_logger.LogInformation("LoadCacheFromStorage: Finished LoadCacheFromStorage");
}
/// <inheritdoc/>
public IStorageContainerClientSleeve GetBlobContainerSleeveForUri(Uri uri, StorageClientProviderContext context)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri));
}
if (context == null)
{
throw new ArgumentNullException(
nameof(context),
$"Use {nameof(StorageClientProviderContext.None)} instead of null for 'empty' context");
}
// Cache policy notes -- see commentary in Gridwich.Core/Models/BlobBaseClientProvider.cs
var cacheKey = GetCacheKeyForBlobContainerClient(uri, context);
var sleeve = _blobContainerClientCache.Get <IStorageContainerClientSleeve>(cacheKey);
if (sleeve != null)
{
// found one, fix up it's context values to the new ones.
// Note that the fact that we found one means the ClientRequestID is already correct, but
// we should update the ETag & flag information, thus the ResetTo().
sleeve.Context.ResetTo(context);
}
else
{
sleeve = CreateBlobContainerClientForUri(uri, context);
_blobContainerClientCache.Add(cacheKey, sleeve, ClientExpirationTime);
}
return(sleeve);
}
public virtual void Add_DefaultTestObject_ThrowsException()
{
Assert.Multiple(() =>
{
var ex = Assert.Throws <ArgumentNullException>(() =>
{
CachingService.Add(Fixture.Create <string>(), default(TestObject));
});
Assert.That(ex?.Message, Is.EqualTo("Value cannot be null.\r\nParameter name: item"));
});
}
public async Task <List <Book> > GetAll()
{
var books = _cache.Get <List <Book> >("books_in_cache");
if (books == null)
{
books = await _repository.GetAll().ConfigureAwait(false);
_cache.Add("books_in_cache", books);
}
return(await Task.FromResult(books).ConfigureAwait(false));
}
private void UpdateDiaspora(Ballot ballot, IGrouping <string, CsvTurnout> diasporaTurnouts,
ApplicationDbContext dbContext,
List <Turnout> existingTurnouts)
{
var totalDiasporaVotes = 0;
foreach (var csvTurnout in diasporaTurnouts.GroupBy(d => d.UAT))
{
var csvTurnouts = csvTurnout.ToList();
var turnout = csvTurnouts.FirstOrDefault();
var dbCountry = FindCountry(turnout);
if (dbCountry == null)
{
Console.WriteLine($"{turnout.UAT} not found in the database");
continue;
}
var countryTurnout = existingTurnouts
.FirstOrDefault(t => t.Division == ElectionDivision.Diaspora_Country && t.CountryId == dbCountry.Id);
if (countryTurnout == null)
{
countryTurnout = new Turnout
{
Division = ElectionDivision.Diaspora_Country,
CountryId = dbCountry.Id,
BallotId = ballot.BallotId
};
}
countryTurnout.EligibleVoters = csvTurnouts.Sum(c => c.EnrolledVoters + c.ComplementaryList);
countryTurnout.TotalVotes = csvTurnouts.Sum(c => c.TotalVotes);
totalDiasporaVotes += countryTurnout.TotalVotes;
dbContext.Update(countryTurnout);
}
var diasporaTurnout = existingTurnouts
.FirstOrDefault(t => t.Division == ElectionDivision.Diaspora);
if (diasporaTurnout == null)
{
diasporaTurnout = new Turnout
{
Division = ElectionDivision.Diaspora,
BallotId = ballot.BallotId,
TotalVotes = totalDiasporaVotes
};
}
_appCache.Add(MemoryCache.DiasporaTurnout, diasporaTurnout, DateTimeOffset.Now.AddMinutes(120));
diasporaTurnout.TotalVotes = totalDiasporaVotes;
dbContext.Update(diasporaTurnout);
}
public void RefreshData()
{
try
{
if (string.IsNullOrEmpty(_secretManager.Secrets.GoogleApiKey))
{
return;
}
if (DateTime.Now.Subtract(_lastAccessed).Minutes < 30)
{
return;
}
_lastAccessed = DateTime.Now;
var data = _httpWrapper.GetData(
string.Format(API_URL, _secretManager.Secrets.GoogleApiKey),
ResponseBuilder);
_cache.Add(CACHE_KEY, data,
new MemoryCacheEntryOptions()
{
AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(1)
});
_fileSystemWrapper.SaveFile(FILE_CACHE_PATH, JsonConvert.SerializeObject(data));
}
catch (Exception ex)
{
_logger.LogError(ex, "Error thrown getting data from YouTube");
}
}
///<inheritdoc/>
public async Task <ContainerRegistryInfo> GetContainerRegistryInfoAsync(string imageName)
{
var containerRegistryInfo = cache.Get <ContainerRegistryInfo>(imageName);
if (containerRegistryInfo == null)
{
containerRegistryInfo = await azureProxy.GetContainerRegistryInfoAsync(imageName);
if (containerRegistryInfo != null)
{
cache.Add(imageName, containerRegistryInfo, DateTimeOffset.Now.AddHours(1));
}
}
return(containerRegistryInfo);
}
/// <inheritdoc/>
public async Task <ContainerRegistryInfo> GetContainerRegistryInfoAsync(string imageName)
{
var containerRegistryInfo = cache.Get <ContainerRegistryInfo>(imageName);
if (containerRegistryInfo is null)
{
containerRegistryInfo = await asyncRetryPolicy.ExecuteAsync(() => azureProxy.GetContainerRegistryInfoAsync(imageName));
if (containerRegistryInfo is not null)
{
cache.Add(imageName, containerRegistryInfo, DateTimeOffset.Now.AddHours(1));
}
}
return(containerRegistryInfo);
}
public virtual DateTime GetLastTradingDate()
{
var lastTradingDate = _cache.Get <DateTime>(CacheNames.LastTradingDateCacheKey);
if (lastTradingDate.Date != DateTime.Now.Date)
{
lastTradingDate = DateTime.Now;
if (lastTradingDate.DayOfWeek == DayOfWeek.Sunday)
{
lastTradingDate = lastTradingDate.AddDays(-2);
}
if (lastTradingDate.DayOfWeek == DayOfWeek.Saturday)
{
lastTradingDate = lastTradingDate.AddDays(-1);
}
//consider holiday
_cache.Add(CacheNames.LastTradingDateCacheKey, lastTradingDate.Date, NextClosingDateTime());
}
return(lastTradingDate.Date);
}
public async Task <int> GetUserId()
{
if (_appCache.Contains(UserIdKey))
{
return(_appCache.Get <int>(UserIdKey));
}
int userId;
var user = await _userService.GetByUserNameAsync(UserName);
if (user == null)
{
// No user exists so let's create one
user = new Data.Models.User
{
Name = UserName,
UserName = UserName
};
userId = await _userService.CreateAsync(user /*, _appConfigHelper.SystemUserId*/);
}
else
{
userId = user.Id;
}
_appCache.Add(UserIdKey, userId);
return(userId);
}
public Task AddAsync <TKey, TValue>(
TKey cacheKey,
TValue item) where TKey : CacheKey <TValue>
{
_appCache.Add(cacheKey, item, _memoryCacheEntryOptions);
return(Task.CompletedTask);
}
private async Task RefreshVMSizesAndPricesAsync() // TODO: implement
{
logger.LogInformation("VM list cache refresh call to Azure started.");
var virtualMachineInfos = await azureProxy.GetVmSizesAndPricesAsync();
cache.Add("vmSizesAndPrices", virtualMachineInfos, DateTimeOffset.MaxValue);
logger.LogInformation("VM list cache refresh call to Azure completed.");
}
public void Set <T>(string key, T value, TimeSpan expiry)
{
Remove(key);
if (value != null)
{
_cache.Add(key, value, expiry);
}
}
public static async Task <string> GetImageAsync(string url)
{
var base64 = cache.Get <string>(url);
if (!string.IsNullOrEmpty(base64))
{
return(base64);
}
using (WebClient webClient = new WebClient())
{
byte[] dataArr = webClient.DownloadData(url);
//save file to local
base64 = Convert.ToBase64String(dataArr);
cache.Add <string>(url, base64);
return(base64);
}
}
/// <summary>
/// Add item to cache
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="cache"></param>
/// <param name="key"></param>
/// <param name="item"></param>
public static void Add <T>(this IAppCache cache, string key, T item)
{
if (cache == null)
{
throw new ArgumentNullException(nameof(cache));
}
cache.Add(key, item, cache.DefaultCachePolicy.BuildOptions());
}
public void GetBlobContainerClientForUri_ShouldReturnNewClient_WhenNotInDictionary()
{
// Arrange
var context = new StorageClientProviderContext("{ \"something\" : \"good\" }", false, string.Empty);
var expectedScheme = "https";
var expectedAccount = "gridwichasset00sasb";
var expectedEndpointSuffix = "blob.core.windows.net";
var expectedContainer = "container1";
Uri uri1 = new Uri($"{expectedScheme}://{expectedAccount}.{expectedEndpointSuffix}/{expectedContainer}");
Uri uri2 = new Uri($"{expectedScheme}://{expectedAccount}.{expectedEndpointSuffix}/container2");
var key1 = BlobContainerClientProvider.GetCacheKeyForBlobContainerClient(uri1, context);
var key2 = BlobContainerClientProvider.GetCacheKeyForBlobContainerClient(uri2, context);
// Ensure we don't pick up a cached object:
_blobContainerClientCache.Remove(key1);
_blobContainerClientCache.Remove(key2);
var blobContainerClient = Mock.Of <BlobContainerClient>();
var sleeve = MockSleeve(blobContainerClient, context);
_blobContainerClientCache.Add(key1, sleeve);
// Act
var existingSleeve = _blobContainerClientProvider.GetBlobContainerSleeveForUri(uri1, context);
var newSleeve = _blobContainerClientProvider.GetBlobContainerSleeveForUri(uri2, context);
// Assert
// Existing should match original and new shouldn't.
newSleeve.Client.ShouldBeOfType <BlobContainerClient>();
newSleeve.Client.ShouldNotBeSameAs(blobContainerClient);
newSleeve.Context.ShouldBeEquivalentTo(context);
existingSleeve.Client.ShouldBeAssignableTo <BlobContainerClient>();
existingSleeve.Client.ShouldBeSameAs(blobContainerClient);
existingSleeve.Context.ShouldBeEquivalentTo(context);
newSleeve.ShouldNotBe(sleeve);
newSleeve.ShouldNotBe(existingSleeve);
existingSleeve.ShouldNotBe(newSleeve);
existingSleeve.ShouldBe(sleeve);
}
/// <summary>
/// Agrega una entrada al caché.
/// </summary>
/// <typeparam name="TEntry">El tipo de la entrada a guardar.</typeparam>
/// <param name="key">La clave que identifica la entrada a guardar.</param>
/// <param name="item">El valor o información de la enterada que se desea guardar.</param>
internal static void Add <TEntry>(string key, TEntry item)
{
if (Policy == CachePolicy.BypassCache)
{
return;
}
cache.Add(key, item, cacheOptions);
ServiceLocator.Instance.LoggingProvider.WriteDebug($"CacheStore => Entry: '{key}' saved to cache.");
}
public async Task ReturnsCachedWhenPopulated()
{
cache.Add(code, new Airport {
});
var result = await sut.GetAirportData(code);
Assert.AreEqual(cache.Get <Airport>(code), result);
mock.Verify(p => p.GetAirportData(It.IsAny <string>()), Times.Never);
}
/// <inheritdoc/>
public IStorageBlobClientSleeve GetBlobClientSleeveForUri(Uri blobUri, StorageClientProviderContext context)
{
if (blobUri == null)
{
throw new ArgumentNullException(nameof(blobUri));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
// Cache policy
//
// We need to support sequences of related storage operations. For example, in Gridwich,
// deleting a blob requires two sequential steps: retrieving metadata and then deleting the blob. If
// we permit caching keyed solely on blob URI, there arise issues with "crossing the streams" -- i.e.,
// two request streams both happen to target the same blob at the overlapping times. Specifically,
// retrieving a client primed for a different Operation Context/ETag combination causes failures.
//
// Another consideration is that the Storage Service, the consumer of the cache items, is not
// structured to pass clients from method to method. In general, if one storage service method
// calls another (e.g., DeleteBlob calls GetMetadata), it passed only the URI and context, not the client.
// Thus it is important that it be possible for those two to be sufficient for each method to retrieve
// a "hot" client instance from the cache (or maybe a fresh one, the first time), avoiding the need for
// using a fresh client each time and incurring authentication, connection establishment, etc.
//
// For those reasons, two policies/conventions:
// 1. The client cache will always be keyed solely on the blobURI/Context combination.
// Should some valid circumstance arise where a context is not available, code falls back to using
// StorageClientProviderContext.NO_CONTEXT.
// 2. The cache item expiry time will be dropped to a much lower time span than the original 10 minutes.
// Cache items should persist for as close to the duration of a request. Since Storage service
// methods are unaware of how callers are sequencing them, it is not clear when it is "safe" for
// a method to remove cache items. Thus, we will have to depend on cache item expiry to keep
// the cache size down.
var cacheKey = GetCacheKeyForBlobBaseClient(blobUri, context);
var sleeve = _blobBaseClientCache.Get <IStorageBlobClientSleeve>(cacheKey);
if (sleeve != null)
{
// found one, fix up it's context values to the new ones.
// Note that the fact that we found one means the ClientRequestID is already correct, but
// we should update the ETag & flag information, thus the ResetTo().
sleeve.Context.ResetTo(context);
}
else
{
sleeve = CreateBlobClientSleeveForUri(blobUri, context);
_blobBaseClientCache.Add(cacheKey, sleeve, ClientExpirationTime);
}
return(sleeve);
}
/// <summary>
/// Add item to cache
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="cache"></param>
/// <param name="key"></param>
/// <param name="item"></param>
/// <param name="slidingExpiration"></param>
public static void Add <T>(this IAppCache cache, string key, T item, TimeSpan slidingExpiration)
{
if (cache == null)
{
throw new ArgumentNullException(nameof(cache));
}
cache.Add(key, item, new MemoryCacheEntryOptions {
SlidingExpiration = slidingExpiration
});
}
/// <summary>
/// Add item to cache
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="cache"></param>
/// <param name="key"></param>
/// <param name="item"></param>
/// <param name="expires"></param>
public static void Add <T>(this IAppCache cache, string key, T item, DateTimeOffset expires)
{
if (cache == null)
{
throw new ArgumentNullException(nameof(cache));
}
cache.Add(key, item, new MemoryCacheEntryOptions {
AbsoluteExpiration = expires
});
}
public void GetIEventGridClientForTopic_ShouldReturnNewClient_WhenNotInDictionary()
{
// Arrange
const string topic1Name = "TOPIC1_NAME";
const string topic1Key = "TOPIC1_KEY";
const string topic2Name = "TOPIC2_NAME";
const string topic2Key = "TOPIC2_KEY";
var eventGridClient = Mock.Of <IEventGridClient>();
_eventGridClientCache.Add(EventGridClientProvider.GetCacheKeyForEventGridClient(topic1Name, topic1Key), eventGridClient);
// Act
var newClient = _eventGridClientProvider.GetEventGridClientForTopic(topic2Name, topic2Key);
var existingClient = _eventGridClientProvider.GetEventGridClientForTopic(topic1Name, topic1Key);
// Assert
newClient.ShouldBeOfType <EventGridClient>();
newClient.SerializationSettings.ContractResolver.ShouldBeEquivalentTo(JsonHelpers.GridwichSerializerSettings.ContractResolver);
newClient.ShouldNotBe(eventGridClient);
existingClient.ShouldBe(eventGridClient);
}
public void Setup()
{
ComplexObject = new ComplexObject();
MemCache = new MemoryCache(new MemoryCacheOptions());
PopulatedMemCache = new MemoryCache(new MemoryCacheOptions());
AppCache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions())));
PopulatedAppCache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions())));
PopulatedAppCache.Add(CacheKey, ComplexObject);
PopulatedMemCache.Set(CacheKey, ComplexObject);
}
internal string GetAuthToken()
{
const string key = nameof(GetAuthToken);
var value = _cache.Get <AccessTokenResponse>(key);
if (value == null)
{
var tokenData = FetchAuthTokenFromServer();
_cache.Add(key, tokenData, DateTimeOffset.Now.AddSeconds(tokenData.expires_in - 60));
}
return(_cache.Get <AccessTokenResponse>(key).access_token);
}
public async Task <Airport> GetAirportData(string code)
{
var result = await cacheService.GetAsync <Airport>(code);
if (result == null)
{
result = await client.GetAirportData(code);
cacheService.Add(code, result, TimeSpan.FromDays(1));
}
return(result);
}
public void GetBlobBaseClientForUri_ShouldReturnNewClient_WhenNotInDictionary()
{
// Arrange
var context = new StorageClientProviderContext("{ \"something\" : \"good\" }", false, string.Empty);
var expectedScheme = "https";
var expectedAccount = "gridwichasset00sasb";
var expectedEndpointSuffix = "core.windows.net";
var expectedContainer = "container1";
var expectedBlob = "path1/blob1.mp4";
Uri uri1 = new Uri($"{expectedScheme}://{expectedAccount}.{expectedEndpointSuffix}/{expectedContainer}/{expectedBlob}");
Uri uri2 = new Uri("https://gridwichasset00sasb.blob.core.windows.net/container2/blob2.mp4");
// Ensure we don't pick up a cached object:
_blobBaseClientCache.Remove(BlobBaseClientProvider.GetCacheKeyForBlobBaseClient(uri1, context));
_blobBaseClientCache.Remove(BlobBaseClientProvider.GetCacheKeyForBlobBaseClient(uri2, context));
var blobBaseClient = Mock.Of <BlobBaseClient>();
var sleeve = MockSleeve(blobBaseClient, context);
_blobBaseClientCache.Add(BlobBaseClientProvider.GetCacheKeyForBlobBaseClient(uri1, context), sleeve);
// Act
var existingClientSleeve = _blobBaseClientProvider.GetBlobClientSleeveForUri(uri1, context);
var existingClient = existingClientSleeve.Client;
var newClientSleeve = _blobBaseClientProvider.GetBlobClientSleeveForUri(uri2, context);
var newClient = newClientSleeve.Client;
// Assert
// So existing should match original and new shouldn't.
existingClientSleeve.ShouldNotBe(newClientSleeve);
existingClientSleeve.ShouldBe(sleeve);
newClient.ShouldBeOfType <BlobBaseClient>();
newClient.ShouldNotBeSameAs(sleeve.Client);
existingClient.ShouldBe(sleeve.Client);
newClient.ShouldNotBe(existingClient);
}
public async Task <IActionResult> OnPost()
{
Language = LanguageProvider.GetValidatedLanguage(_userService.ClaimsPrincipalToAuthenticatedUser(User), Request);
var forum = await _context.PhpbbForums.AsNoTracking().FirstOrDefaultAsync(filter => filter.ForumId == ForumId);
if (forum == null)
{
return(RedirectToPage("Error", new { IsNotFound = true }));
}
if (string.IsNullOrWhiteSpace(Password))
{
ModelState.AddModelError(nameof(Password), LanguageProvider.Errors[Language, "MISSING_REQUIRED_FIELD"]);
return(Page());
}
if (forum == null)
{
return(RedirectToPage("Error", new { isNotFound = true }));
}
if (string.IsNullOrWhiteSpace(forum.ForumPassword))
{
if (string.IsNullOrWhiteSpace(ReturnUrl))
{
return(RedirectToPage("ViewForum", new { ForumId }));
}
else
{
return(Redirect(HttpUtility.UrlDecode(ReturnUrl)));
}
}
if (forum.ForumPassword != Crypter.Phpass.Crypt(Password, forum.ForumPassword))
{
ModelState.AddModelError(nameof(Password), LanguageProvider.Errors[Language, "WRONG_PASS"]);
return(await OnGet());
}
else
{
var userId = _userService.ClaimsPrincipalToAuthenticatedUser(User)?.UserId ?? Constants.ANONYMOUS_USER_ID;
_cache.Add(_utils.GetForumLoginCacheKey(userId, ForumId), 1, TimeSpan.FromMinutes(30));
if (string.IsNullOrWhiteSpace(ReturnUrl))
{
return(RedirectToPage("ViewForum", new { ForumId }));
}
return(Redirect(HttpUtility.UrlDecode(ReturnUrl)));
}
}
/// <summary>
/// Requests data for current region from game api.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="requestUri"></param>
/// <param name="namespace"></param>
/// <param name="requestOptions"></param>
/// <returns></returns>
public async Task <T> GetAsync <T>(string requestUri, string @namespace, RequestOptions requestOptions = null)
{
// Use default region and locale if not specified in request scope
var region = requestOptions?.Region ?? _defaultApiConfiguration.Region;
var locale = requestOptions.UseRegionLocale
? region.GetDefaultLocale()
: (requestOptions?.Locale ?? _defaultApiConfiguration.Locale);
var useCache = requestOptions?.UseCache ?? _defaultApiConfiguration.UseCache;
var cacheDuration = requestOptions?.CacheDuration ?? _defaultApiConfiguration.CacheDuration;
// Refresh token
var token = await RequestTokenAsync(region);
// Build query string
var queryParams = requestOptions?.QueryParams ?? new Dictionary <string, string>();
queryParams.Add("locale", locale.ToQueryString());
if (!string.IsNullOrEmpty(@namespace))
{
queryParams.Add("namespace", $"{@namespace}-{region.ToQueryString()}");
}
var finalUri = QueryHelpers.AddQueryString($"{_defaultApiConfiguration.GetGameApiUrl(region)}{requestUri}", queryParams);
// Setup request authorization
_gameApiClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token.AccessToken);
// Send request
var response = useCache ? await _cache.GetAsync <string>(finalUri) : null;
if (response == null)
{
using var responseMessage = await _gameApiClient.GetAsync(finalUri);
responseMessage.EnsureSuccessStatusCode();
response = await responseMessage.Content.ReadAsStringAsync();
if (useCache)
{
_cache.Add(finalUri, response, TimeSpan.FromSeconds(cacheDuration));
}
}
// Return response
return(JsonConvert.DeserializeObject <T>(response));
}
public void RefreshData()
{
try
{
var data = _httpWrapper.GetData(API_URL, ResponseBuilder);
_cache.Add(CACHE_KEY, data,
new MemoryCacheEntryOptions()
{
AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(1)
});
_fileSystemWrapper.SaveFile(FILE_CACHE_PATH, JsonConvert.SerializeObject(data));
}
catch (Exception ex)
{
_logger.LogError(ex, "Error thrown getting data from Meetup");
}
}
public async Task <Evaluation> Submit(Guid profileId, Guid chapterId, int pageIndex, string solution)
{
var profile = await _profiles.Get(profileId, UserId);
var submit = new Submit(chapterId, pageIndex, solution);
var evaluation = await _evaluator.Evaluate(profile, submit);
if (!profile.Equals(evaluation.Profile))
{
var cachedId = profile.Id.ToString();
_cache.Remove(cachedId);
_cache.Add(cachedId, profile);
}
return(evaluation);
}