public virtual async Task CanUseScopedCaches() { var cache = GetCacheClient(); if (cache == null) { return; } using (cache) { await cache.RemoveAllAsync(); var scopedCache1 = new ScopedCacheClient(cache, "scoped1"); var nestedScopedCache1 = new ScopedCacheClient(scopedCache1, "nested"); var scopedCache2 = new ScopedCacheClient(cache, "scoped2"); await cache.SetAsync("test", 1); await scopedCache1.SetAsync("test", 2); await nestedScopedCache1.SetAsync("test", 3); Assert.Equal(1, (await cache.GetAsync <int>("test")).Value); Assert.Equal(2, (await scopedCache1.GetAsync <int>("test")).Value); Assert.Equal(3, (await nestedScopedCache1.GetAsync <int>("test")).Value); Assert.Equal(3, (await scopedCache1.GetAsync <int>("nested:test")).Value); Assert.Equal(3, (await cache.GetAsync <int>("scoped1:nested:test")).Value); // ensure GetAllAsync returns unscoped keys Assert.Equal("test", (await scopedCache1.GetAllAsync <int>("test")).Keys.FirstOrDefault()); Assert.Equal("test", (await nestedScopedCache1.GetAllAsync <int>("test")).Keys.FirstOrDefault()); await scopedCache2.SetAsync("test", 1); var result = await scopedCache1.RemoveByPrefixAsync(String.Empty); Assert.Equal(2, result); // delete without any matching keys result = await scopedCache1.RemoveByPrefixAsync(String.Empty); Assert.Equal(0, result); Assert.False((await scopedCache1.GetAsync <int>("test")).HasValue); Assert.False((await nestedScopedCache1.GetAsync <int>("test")).HasValue); Assert.Equal(1, (await cache.GetAsync <int>("test")).Value); Assert.Equal(1, (await scopedCache2.GetAsync <int>("test")).Value); await scopedCache2.RemoveAllAsync(); Assert.False((await scopedCache1.GetAsync <int>("test")).HasValue); Assert.False((await nestedScopedCache1.GetAsync <int>("test")).HasValue); Assert.False((await scopedCache2.GetAsync <int>("test")).HasValue); Assert.Equal(1, (await cache.GetAsync <int>("test")).Value); } }
protected override async Task <JobResult> ProcessQueueEntryAsync(QueueEntryContext <NetInfoWorkItem> context) { try { NetInfoWorkItem block = context.QueueEntry.Value; _logger.LogDebug("NetStatsJob fired, Block: {@Block}", block); NetInfo netInfo = await _statsCache.GetAsync(NetInfo.DefaultCacheKey, new NetInfo()).AnyContext(); netInfo.BestBlock = block.BlockNumber; netInfo.BestBlockTimestamp = block.Timestamp; netInfo.Difficulty = await _netInfoService.GetDifficulty(block.Difficulty).AnyContext(); netInfo.AverageBlockTime = await _netInfoService.GetAverageBlockTime(block.Timestamp).AnyContext(); netInfo.AverageNetworkHashRate = await _netInfoService.GetAverageNetworkHashRate(block.Difficulty).AnyContext(); netInfo.ConnectedPeerCount = await _netInfoService.GetConnectedPeerCount().AnyContext(); await _statsCache.SetAsync(NetInfo.DefaultCacheKey, netInfo).AnyContext(); } catch (Exception e) { return(JobResult.FromException(e)); } return(JobResult.Success); }
public async Task <HealthCheckResult> CheckCacheAsync() { var sw = Stopwatch.StartNew(); try { var cache = new ScopedCacheClient(_configuration.Cache, "health"); var cacheValue = await cache.GetAsync <string>("__PING__").AnyContext(); if (cacheValue.HasValue) { return(HealthCheckResult.NotHealthy("Cache Not Working")); } } catch (Exception ex) { return(HealthCheckResult.NotHealthy("Cache Not Working: " + ex.Message)); } finally { sw.Stop(); _logger.LogTrace("Checking cache took {Duration:g}", sw.Elapsed); } return(HealthCheckResult.Healthy); }
public async Task <HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { var sw = Stopwatch.StartNew(); try { var cache = new ScopedCacheClient(_cache, "health"); var cacheValue = await cache.GetAsync <string>("__PING__").AnyContext(); if (cacheValue.HasValue) { return(HealthCheckResult.Unhealthy("Cache Not Working")); } } catch (Exception ex) { return(HealthCheckResult.Unhealthy("Cache Not Working.", ex)); } finally { sw.Stop(); _logger.LogTrace("Checking cache took {Duration:g}", sw.Elapsed); } return(HealthCheckResult.Healthy()); }
protected override async Task <JobResult> ProcessQueueEntryAsync(QueueEntryContext <WebHookNotification> context) { var body = context.QueueEntry.Value; bool shouldLog = body.ProjectId != _appOptions.Value.InternalProjectId; using (_logger.BeginScope(new ExceptionlessState().Organization(body.OrganizationId).Project(body.ProjectId))) { if (shouldLog) { _logger.LogTrace("Process web hook call: id={Id} project={1} url={Url}", context.QueueEntry.Id, body.ProjectId, body.Url); } if (!await IsEnabledAsync(body).AnyContext()) { _logger.LogInformation("Web hook cancelled: Web hook is disabled"); return(JobResult.Cancelled); } var cache = new ScopedCacheClient(_cacheClient, GetCacheKeyScope(body)); long consecutiveErrors = await cache.GetAsync <long>(ConsecutiveErrorsCacheKey, 0).AnyContext(); if (consecutiveErrors > 10) { var lastAttempt = await cache.GetAsync(LastAttemptCacheKey, SystemClock.UtcNow).AnyContext(); var nextAttemptAllowedAt = lastAttempt.AddMinutes(15); if (nextAttemptAllowedAt >= SystemClock.UtcNow) { _logger.LogInformation("Web hook cancelled due to {FailureCount} consecutive failed attempts. Will be allowed to try again at {NextAttempt}.", consecutiveErrors, nextAttemptAllowedAt); return(JobResult.Cancelled); } } bool successful = true; HttpResponseMessage response = null; try { using (var timeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5))) { using (var postCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, timeoutCancellationTokenSource.Token)) { response = await _client.PostAsJsonAsync(body.Url, body.Data.ToJson(Formatting.Indented, _jsonSerializerSettings), postCancellationTokenSource.Token).AnyContext(); if (!response.IsSuccessStatusCode) { successful = false; } else if (consecutiveErrors > 0) { await cache.RemoveAllAsync(_cacheKeys).AnyContext(); } } } } catch (OperationCanceledException ex) { successful = false; if (shouldLog) { _logger.LogError(ex, "Timeout calling web hook: status={Status} org={organization} project={project} url={Url}", response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url); } return(JobResult.Cancelled); } catch (Exception ex) { successful = false; if (shouldLog) { _logger.LogError(ex, "Error calling web hook: status={Status} org={organization} project={project} url={Url}", response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url); } return(JobResult.FromException(ex)); } finally { if (successful) { _logger.LogInformation("Web hook POST complete: status={Status} org={organization} project={project} url={Url}", response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url); } else if (response != null && (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Gone)) { _logger.LogWarning("Disabling Web hook instance {WebHookId} due to status code: status={Status} org={organization} project={project} url={Url}", body.Type == WebHookType.Slack ? "Slack" : body.WebHookId, response.StatusCode, body.OrganizationId, body.ProjectId, body.Url); await DisableIntegrationAsync(body).AnyContext(); await cache.RemoveAllAsync(_cacheKeys).AnyContext(); } else { var now = SystemClock.UtcNow; await cache.SetAsync(LastAttemptCacheKey, now, TimeSpan.FromDays(3)).AnyContext(); consecutiveErrors = await cache.IncrementAsync(ConsecutiveErrorsCacheKey, TimeSpan.FromDays(3)).AnyContext(); DateTime firstAttempt; if (consecutiveErrors == 1) { await cache.SetAsync(FirstAttemptCacheKey, now, TimeSpan.FromDays(3)).AnyContext(); firstAttempt = now; } else { firstAttempt = await cache.GetAsync(FirstAttemptCacheKey, now).AnyContext(); } if (consecutiveErrors >= 10) { // don't retry any more context.QueueEntry.MarkCompleted(); // disable if more than 10 consecutive errors over the course of multiple days if (firstAttempt.IsBefore(now.SubtractDays(2))) { _logger.LogWarning("Disabling Web hook instance {WebHookId} due to too many consecutive failures.", body.Type == WebHookType.Slack ? "Slack" : body.WebHookId); await DisableIntegrationAsync(body).AnyContext(); await cache.RemoveAllAsync(_cacheKeys).AnyContext(); } } } } } return(JobResult.Success); }