public virtual async Task CanUseScopedCaches()
{
var cache = GetCacheClient();
if (cache == null)
{
return;
}
using (cache) {
await cache.RemoveAllAsync();
var scopedCache1 = new ScopedCacheClient(cache, "scoped1");
var nestedScopedCache1 = new ScopedCacheClient(scopedCache1, "nested");
var scopedCache2 = new ScopedCacheClient(cache, "scoped2");
await cache.SetAsync("test", 1);
await scopedCache1.SetAsync("test", 2);
await nestedScopedCache1.SetAsync("test", 3);
Assert.Equal(1, (await cache.GetAsync <int>("test")).Value);
Assert.Equal(2, (await scopedCache1.GetAsync <int>("test")).Value);
Assert.Equal(3, (await nestedScopedCache1.GetAsync <int>("test")).Value);
Assert.Equal(3, (await scopedCache1.GetAsync <int>("nested:test")).Value);
Assert.Equal(3, (await cache.GetAsync <int>("scoped1:nested:test")).Value);
// ensure GetAllAsync returns unscoped keys
Assert.Equal("test", (await scopedCache1.GetAllAsync <int>("test")).Keys.FirstOrDefault());
Assert.Equal("test", (await nestedScopedCache1.GetAllAsync <int>("test")).Keys.FirstOrDefault());
await scopedCache2.SetAsync("test", 1);
var result = await scopedCache1.RemoveByPrefixAsync(String.Empty);
Assert.Equal(2, result);
// delete without any matching keys
result = await scopedCache1.RemoveByPrefixAsync(String.Empty);
Assert.Equal(0, result);
Assert.False((await scopedCache1.GetAsync <int>("test")).HasValue);
Assert.False((await nestedScopedCache1.GetAsync <int>("test")).HasValue);
Assert.Equal(1, (await cache.GetAsync <int>("test")).Value);
Assert.Equal(1, (await scopedCache2.GetAsync <int>("test")).Value);
await scopedCache2.RemoveAllAsync();
Assert.False((await scopedCache1.GetAsync <int>("test")).HasValue);
Assert.False((await nestedScopedCache1.GetAsync <int>("test")).HasValue);
Assert.False((await scopedCache2.GetAsync <int>("test")).HasValue);
Assert.Equal(1, (await cache.GetAsync <int>("test")).Value);
}
}
protected override async Task <JobResult> ProcessQueueEntryAsync(QueueEntryContext <NetInfoWorkItem> context)
{
try
{
NetInfoWorkItem block = context.QueueEntry.Value;
_logger.LogDebug("NetStatsJob fired, Block: {@Block}", block);
NetInfo netInfo = await _statsCache.GetAsync(NetInfo.DefaultCacheKey, new NetInfo()).AnyContext();
netInfo.BestBlock = block.BlockNumber;
netInfo.BestBlockTimestamp = block.Timestamp;
netInfo.Difficulty = await _netInfoService.GetDifficulty(block.Difficulty).AnyContext();
netInfo.AverageBlockTime = await _netInfoService.GetAverageBlockTime(block.Timestamp).AnyContext();
netInfo.AverageNetworkHashRate = await _netInfoService.GetAverageNetworkHashRate(block.Difficulty).AnyContext();
netInfo.ConnectedPeerCount = await _netInfoService.GetConnectedPeerCount().AnyContext();
await _statsCache.SetAsync(NetInfo.DefaultCacheKey, netInfo).AnyContext();
}
catch (Exception e)
{
return(JobResult.FromException(e));
}
return(JobResult.Success);
}
public async Task <HealthCheckResult> CheckCacheAsync()
{
var sw = Stopwatch.StartNew();
try {
var cache = new ScopedCacheClient(_configuration.Cache, "health");
var cacheValue = await cache.GetAsync <string>("__PING__").AnyContext();
if (cacheValue.HasValue)
{
return(HealthCheckResult.NotHealthy("Cache Not Working"));
}
} catch (Exception ex) {
return(HealthCheckResult.NotHealthy("Cache Not Working: " + ex.Message));
} finally {
sw.Stop();
_logger.LogTrace("Checking cache took {Duration:g}", sw.Elapsed);
}
return(HealthCheckResult.Healthy);
}
public async Task <HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
{
var sw = Stopwatch.StartNew();
try {
var cache = new ScopedCacheClient(_cache, "health");
var cacheValue = await cache.GetAsync <string>("__PING__").AnyContext();
if (cacheValue.HasValue)
{
return(HealthCheckResult.Unhealthy("Cache Not Working"));
}
} catch (Exception ex) {
return(HealthCheckResult.Unhealthy("Cache Not Working.", ex));
} finally {
sw.Stop();
_logger.LogTrace("Checking cache took {Duration:g}", sw.Elapsed);
}
return(HealthCheckResult.Healthy());
}
protected override async Task <JobResult> ProcessQueueEntryAsync(QueueEntryContext <WebHookNotification> context)
{
var body = context.QueueEntry.Value;
bool shouldLog = body.ProjectId != _appOptions.Value.InternalProjectId;
using (_logger.BeginScope(new ExceptionlessState().Organization(body.OrganizationId).Project(body.ProjectId))) {
if (shouldLog)
{
_logger.LogTrace("Process web hook call: id={Id} project={1} url={Url}", context.QueueEntry.Id, body.ProjectId, body.Url);
}
if (!await IsEnabledAsync(body).AnyContext())
{
_logger.LogInformation("Web hook cancelled: Web hook is disabled");
return(JobResult.Cancelled);
}
var cache = new ScopedCacheClient(_cacheClient, GetCacheKeyScope(body));
long consecutiveErrors = await cache.GetAsync <long>(ConsecutiveErrorsCacheKey, 0).AnyContext();
if (consecutiveErrors > 10)
{
var lastAttempt = await cache.GetAsync(LastAttemptCacheKey, SystemClock.UtcNow).AnyContext();
var nextAttemptAllowedAt = lastAttempt.AddMinutes(15);
if (nextAttemptAllowedAt >= SystemClock.UtcNow)
{
_logger.LogInformation("Web hook cancelled due to {FailureCount} consecutive failed attempts. Will be allowed to try again at {NextAttempt}.", consecutiveErrors, nextAttemptAllowedAt);
return(JobResult.Cancelled);
}
}
bool successful = true;
HttpResponseMessage response = null;
try {
using (var timeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5))) {
using (var postCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, timeoutCancellationTokenSource.Token)) {
response = await _client.PostAsJsonAsync(body.Url, body.Data.ToJson(Formatting.Indented, _jsonSerializerSettings), postCancellationTokenSource.Token).AnyContext();
if (!response.IsSuccessStatusCode)
{
successful = false;
}
else if (consecutiveErrors > 0)
{
await cache.RemoveAllAsync(_cacheKeys).AnyContext();
}
}
}
} catch (OperationCanceledException ex) {
successful = false;
if (shouldLog)
{
_logger.LogError(ex, "Timeout calling web hook: status={Status} org={organization} project={project} url={Url}", response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url);
}
return(JobResult.Cancelled);
} catch (Exception ex) {
successful = false;
if (shouldLog)
{
_logger.LogError(ex, "Error calling web hook: status={Status} org={organization} project={project} url={Url}", response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url);
}
return(JobResult.FromException(ex));
} finally {
if (successful)
{
_logger.LogInformation("Web hook POST complete: status={Status} org={organization} project={project} url={Url}", response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url);
}
else if (response != null && (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Gone))
{
_logger.LogWarning("Disabling Web hook instance {WebHookId} due to status code: status={Status} org={organization} project={project} url={Url}", body.Type == WebHookType.Slack ? "Slack" : body.WebHookId, response.StatusCode, body.OrganizationId, body.ProjectId, body.Url);
await DisableIntegrationAsync(body).AnyContext();
await cache.RemoveAllAsync(_cacheKeys).AnyContext();
}
else
{
var now = SystemClock.UtcNow;
await cache.SetAsync(LastAttemptCacheKey, now, TimeSpan.FromDays(3)).AnyContext();
consecutiveErrors = await cache.IncrementAsync(ConsecutiveErrorsCacheKey, TimeSpan.FromDays(3)).AnyContext();
DateTime firstAttempt;
if (consecutiveErrors == 1)
{
await cache.SetAsync(FirstAttemptCacheKey, now, TimeSpan.FromDays(3)).AnyContext();
firstAttempt = now;
}
else
{
firstAttempt = await cache.GetAsync(FirstAttemptCacheKey, now).AnyContext();
}
if (consecutiveErrors >= 10)
{
// don't retry any more
context.QueueEntry.MarkCompleted();
// disable if more than 10 consecutive errors over the course of multiple days
if (firstAttempt.IsBefore(now.SubtractDays(2)))
{
_logger.LogWarning("Disabling Web hook instance {WebHookId} due to too many consecutive failures.", body.Type == WebHookType.Slack ? "Slack" : body.WebHookId);
await DisableIntegrationAsync(body).AnyContext();
await cache.RemoveAllAsync(_cacheKeys).AnyContext();
}
}
}
}
}
return(JobResult.Success);
}
