public virtual async Task CanUseScopedCachesAsync() { var cache = GetCacheClient(); if (cache == null) { return; } using (cache) { await cache.RemoveAllAsync(); var scopedCache1 = new ScopedCacheClient(cache, "scoped1"); var nestedScopedCache1 = new ScopedCacheClient(scopedCache1, "nested"); var scopedCache2 = new ScopedCacheClient(cache, "scoped2"); await cache.SetAsync("test", 1); await scopedCache1.SetAsync("test", 2); await nestedScopedCache1.SetAsync("test", 3); Assert.Equal(1, (await cache.GetAsync <int>("test")).Value); Assert.Equal(2, (await scopedCache1.GetAsync <int>("test")).Value); Assert.Equal(3, (await nestedScopedCache1.GetAsync <int>("test")).Value); Assert.Equal(3, (await scopedCache1.GetAsync <int>("nested:test")).Value); Assert.Equal(3, (await cache.GetAsync <int>("scoped1:nested:test")).Value); // ensure GetAllAsync returns unscoped keys Assert.Equal("test", (await scopedCache1.GetAllAsync <int>("test")).Keys.FirstOrDefault()); Assert.Equal("test", (await nestedScopedCache1.GetAllAsync <int>("test")).Keys.FirstOrDefault()); await scopedCache2.SetAsync("test", 1); int result = await scopedCache1.RemoveByPrefixAsync(String.Empty); Assert.Equal(2, result); // delete without any matching keys result = await scopedCache1.RemoveByPrefixAsync(String.Empty); Assert.Equal(0, result); Assert.False((await scopedCache1.GetAsync <int>("test")).HasValue); Assert.False((await nestedScopedCache1.GetAsync <int>("test")).HasValue); Assert.Equal(1, (await cache.GetAsync <int>("test")).Value); Assert.Equal(1, (await scopedCache2.GetAsync <int>("test")).Value); await scopedCache2.RemoveAllAsync(); Assert.False((await scopedCache1.GetAsync <int>("test")).HasValue); Assert.False((await nestedScopedCache1.GetAsync <int>("test")).HasValue); Assert.False((await scopedCache2.GetAsync <int>("test")).HasValue); Assert.Equal(1, (await cache.GetAsync <int>("test")).Value); Assert.Equal(0, await scopedCache1.GetAsync <double>("total", 0)); Assert.Equal(10, await scopedCache1.IncrementAsync("total", 10)); Assert.Equal(10, await scopedCache1.GetAsync <double>("total", 0)); Assert.Equal(0, await nestedScopedCache1.GetAsync <double>("total", 0)); Assert.Equal(20, await nestedScopedCache1.IncrementAsync("total", 20)); Assert.Equal(20, await nestedScopedCache1.GetAsync <double>("total", 0)); Assert.Equal(1, await nestedScopedCache1.RemoveAllAsync(new[] { "id", "total" })); Assert.Equal(0, await nestedScopedCache1.GetAsync <double>("total", 0)); Assert.Equal(1, await scopedCache1.RemoveAllAsync(new[] { "id", "total" })); Assert.Equal(0, await scopedCache1.GetAsync <double>("total", 0)); } }
protected override async Task <JobResult> ProcessQueueEntryAsync(QueueEntryContext <WebHookNotification> context) { var body = context.QueueEntry.Value; bool shouldLog = body.ProjectId != _appOptions.Value.InternalProjectId; using (_logger.BeginScope(new ExceptionlessState().Organization(body.OrganizationId).Project(body.ProjectId))) { if (shouldLog) { _logger.LogTrace("Process web hook call: id={Id} project={1} url={Url}", context.QueueEntry.Id, body.ProjectId, body.Url); } if (!await IsEnabledAsync(body).AnyContext()) { _logger.LogInformation("Web hook cancelled: Web hook is disabled"); return(JobResult.Cancelled); } var cache = new ScopedCacheClient(_cacheClient, GetCacheKeyScope(body)); long consecutiveErrors = await cache.GetAsync <long>(ConsecutiveErrorsCacheKey, 0).AnyContext(); if (consecutiveErrors > 10) { var lastAttempt = await cache.GetAsync(LastAttemptCacheKey, SystemClock.UtcNow).AnyContext(); var nextAttemptAllowedAt = lastAttempt.AddMinutes(15); if (nextAttemptAllowedAt >= SystemClock.UtcNow) { _logger.LogInformation("Web hook cancelled due to {FailureCount} consecutive failed attempts. Will be allowed to try again at {NextAttempt}.", consecutiveErrors, nextAttemptAllowedAt); return(JobResult.Cancelled); } } bool successful = true; HttpResponseMessage response = null; try { using (var timeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5))) { using (var postCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, timeoutCancellationTokenSource.Token)) { response = await _client.PostAsJsonAsync(body.Url, body.Data.ToJson(Formatting.Indented, _jsonSerializerSettings), postCancellationTokenSource.Token).AnyContext(); if (!response.IsSuccessStatusCode) { successful = false; } else if (consecutiveErrors > 0) { await cache.RemoveAllAsync(_cacheKeys).AnyContext(); } } } } catch (OperationCanceledException ex) { successful = false; if (shouldLog) { _logger.LogError(ex, "Timeout calling web hook: status={Status} org={organization} project={project} url={Url}", response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url); } return(JobResult.Cancelled); } catch (Exception ex) { successful = false; if (shouldLog) { _logger.LogError(ex, "Error calling web hook: status={Status} org={organization} project={project} url={Url}", response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url); } return(JobResult.FromException(ex)); } finally { if (successful) { _logger.LogInformation("Web hook POST complete: status={Status} org={organization} project={project} url={Url}", response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url); } else if (response != null && (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Gone)) { _logger.LogWarning("Disabling Web hook instance {WebHookId} due to status code: status={Status} org={organization} project={project} url={Url}", body.Type == WebHookType.Slack ? "Slack" : body.WebHookId, response.StatusCode, body.OrganizationId, body.ProjectId, body.Url); await DisableIntegrationAsync(body).AnyContext(); await cache.RemoveAllAsync(_cacheKeys).AnyContext(); } else { var now = SystemClock.UtcNow; await cache.SetAsync(LastAttemptCacheKey, now, TimeSpan.FromDays(3)).AnyContext(); consecutiveErrors = await cache.IncrementAsync(ConsecutiveErrorsCacheKey, TimeSpan.FromDays(3)).AnyContext(); DateTime firstAttempt; if (consecutiveErrors == 1) { await cache.SetAsync(FirstAttemptCacheKey, now, TimeSpan.FromDays(3)).AnyContext(); firstAttempt = now; } else { firstAttempt = await cache.GetAsync(FirstAttemptCacheKey, now).AnyContext(); } if (consecutiveErrors >= 10) { // don't retry any more context.QueueEntry.MarkCompleted(); // disable if more than 10 consecutive errors over the course of multiple days if (firstAttempt.IsBefore(now.SubtractDays(2))) { _logger.LogWarning("Disabling Web hook instance {WebHookId} due to too many consecutive failures.", body.Type == WebHookType.Slack ? "Slack" : body.WebHookId); await DisableIntegrationAsync(body).AnyContext(); await cache.RemoveAllAsync(_cacheKeys).AnyContext(); } } } } } return(JobResult.Success); }