private async Task <Dictionary <string, double> > GetUserCountByProjectIdsAsync(ICollection <Stack> stacks, AppFilter sf, DateTime utcStart, DateTime utcEnd)
{
var scopedCacheClient = new ScopedCacheClient(_cache, $"Project:user-count:{utcStart.Floor(TimeSpan.FromMinutes(15)).Ticks}-{utcEnd.Floor(TimeSpan.FromMinutes(15)).Ticks}");
var projectIds = stacks.Select(s => s.ProjectId).Distinct().ToList();
var cachedTotals = await scopedCacheClient.GetAllAsync <double>(projectIds);
var totals = cachedTotals.Where(kvp => kvp.Value.HasValue).ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Value);
if (totals.Count == projectIds.Count)
{
return(totals);
}
var systemFilter = new RepositoryQuery <PersistentEvent>().AppFilter(sf).DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date).Index(utcStart, utcEnd);
var projects = cachedTotals.Where(kvp => !kvp.Value.HasValue).Select(kvp => new Project {
Id = kvp.Key, OrganizationId = stacks.FirstOrDefault(s => s.ProjectId == kvp.Key)?.OrganizationId
}).ToList();
var countResult = await _eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(projects.BuildFilter()).AggregationsExpression("terms:(project_id cardinality:user)"));
// Cache all projects that have more than 10 users for 5 minutes.
var projectTerms = countResult.Aggregations.Terms <string>("terms_project_id").Buckets;
var aggregations = projectTerms.ToDictionary(t => t.Key, t => t.Aggregations.Cardinality("cardinality_user").Value.GetValueOrDefault());
await scopedCacheClient.SetAllAsync(aggregations.Where(t => t.Value >= 10).ToDictionary(k => k.Key, v => v.Value), TimeSpan.FromMinutes(5));
totals.AddRange(aggregations);
return(totals);
}
private async Task <Dictionary <string, double> > GetUserCountByProjectIdsAsync(ICollection <Stack> stacks, string systemFilter, DateTime utcStart, DateTime utcEnd)
{
var scopedCacheClient = new ScopedCacheClient(_cacheClient, $"project:user-count:{utcStart.Floor(TimeSpan.FromMinutes(15)).Ticks}-{utcEnd.Floor(TimeSpan.FromMinutes(15)).Ticks}");
var projectIds = stacks.Select(s => s.ProjectId).Distinct().ToList();
var cachedTotals = await scopedCacheClient.GetAllAsync <double>(projectIds);
var totals = cachedTotals.Where(kvp => kvp.Value.HasValue).ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Value);
if (totals.Count == projectIds.Count)
{
return(totals);
}
var projects = cachedTotals.Where(kvp => !kvp.Value.HasValue).Select(kvp => new Project {
Id = kvp.Key, OrganizationId = stacks.FirstOrDefault(s => s.ProjectId == kvp.Key)?.OrganizationId
}).ToList();
var projectTerms = await _eventStats.GetNumbersTermsStatsAsync("project_id", _distinctUsersFields, utcStart, utcEnd, systemFilter, projects.BuildRetentionFilter());
// Cache all projects that have more than 10 users for 5 minutes.
await scopedCacheClient.SetAllAsync(projectTerms.Terms.Where(t => t.Numbers[0] >= 10).ToDictionary(t => t.Term, t => t.Numbers[0]), TimeSpan.FromMinutes(5));
totals.AddRange(projectTerms.Terms.ToDictionary(kvp => kvp.Term, kvp => kvp.Numbers[0]));
return(totals);
}
public virtual async Task CanUseScopedCaches()
{
var cache = GetCacheClient();
if (cache == null)
{
return;
}
using (cache) {
await cache.RemoveAllAsync();
var scopedCache1 = new ScopedCacheClient(cache, "scoped1");
var nestedScopedCache1 = new ScopedCacheClient(scopedCache1, "nested");
var scopedCache2 = new ScopedCacheClient(cache, "scoped2");
await cache.SetAsync("test", 1);
await scopedCache1.SetAsync("test", 2);
await nestedScopedCache1.SetAsync("test", 3);
Assert.Equal(1, (await cache.GetAsync <int>("test")).Value);
Assert.Equal(2, (await scopedCache1.GetAsync <int>("test")).Value);
Assert.Equal(3, (await nestedScopedCache1.GetAsync <int>("test")).Value);
Assert.Equal(3, (await scopedCache1.GetAsync <int>("nested:test")).Value);
Assert.Equal(3, (await cache.GetAsync <int>("scoped1:nested:test")).Value);
// ensure GetAllAsync returns unscoped keys
Assert.Equal("test", (await scopedCache1.GetAllAsync <int>("test")).Keys.FirstOrDefault());
Assert.Equal("test", (await nestedScopedCache1.GetAllAsync <int>("test")).Keys.FirstOrDefault());
await scopedCache2.SetAsync("test", 1);
var result = await scopedCache1.RemoveByPrefixAsync(String.Empty);
Assert.Equal(2, result);
// delete without any matching keys
result = await scopedCache1.RemoveByPrefixAsync(String.Empty);
Assert.Equal(0, result);
Assert.False((await scopedCache1.GetAsync <int>("test")).HasValue);
Assert.False((await nestedScopedCache1.GetAsync <int>("test")).HasValue);
Assert.Equal(1, (await cache.GetAsync <int>("test")).Value);
Assert.Equal(1, (await scopedCache2.GetAsync <int>("test")).Value);
await scopedCache2.RemoveAllAsync();
Assert.False((await scopedCache1.GetAsync <int>("test")).HasValue);
Assert.False((await nestedScopedCache1.GetAsync <int>("test")).HasValue);
Assert.False((await scopedCache2.GetAsync <int>("test")).HasValue);
Assert.Equal(1, (await cache.GetAsync <int>("test")).Value);
}
}
protected AbstractMigrateableService(
TInfo info,
ICacheClient cacheClient,
IMessageBusFactory messageBusFactory
) : base(info, cacheClient, messageBusFactory)
{
AccountStateCache = new ScopedCacheClient(cacheClient, Scopes.MigrationAccountCache);
CharacterStateCache = new ScopedCacheClient(cacheClient, Scopes.MigrationCharacterCache);
MigrationStateCache = new ScopedCacheClient(cacheClient, Scopes.MigrationCache);
}
public static IServiceCollection AddCacheClient(this IServiceCollection services)
{
services.AddSingleton(provider =>
{
var options = provider.GetRequiredService <IOptions <CacheOptions> >().Value;
var type = options.Type.ToLower();
ICacheClient client;
if (type == "auto")
{
var redisOptions = provider.GetRequiredService <IOptions <RedisOptions> >().Value;
type = !string.IsNullOrEmpty(redisOptions.ConnectionString) ? "redis" : "memory";
}
switch (type)
{
case "inmemory":
case "memory":
client = new InMemoryCacheClient(new InMemoryCacheClientOptions
{
LoggerFactory = provider.GetRequiredService <ILoggerFactory>(),
Serializer = provider.GetRequiredService <ISerializer>()
});
break;
case "redis":
client = new RedisCacheClient(new RedisCacheClientOptions
{
ConnectionMultiplexer = provider.GetRequiredService <IConnectionMultiplexer>(),
LoggerFactory = provider.GetRequiredService <ILoggerFactory>(),
Serializer = provider.GetRequiredService <ISerializer>()
});
break;
default:
throw new NotSupportedException($"The cache type {options.Type} is not supported.");
}
if (!string.IsNullOrEmpty(options.Prefix))
{
client = new ScopedCacheClient(client, options.Prefix);
}
return(client);
});
return(services);
}
public AbstractHostedService(
IApplicationLifetime appLifetime,
ICacheClient cache,
IMessageBus messageBus,
IOptions <TInfo> info,
IDataContextFactory dataContextFactory
)
{
_appLifetime = appLifetime;
_cache = cache;
_messageBus = messageBus;
_peers = new ConcurrentDictionary <string, PeerServiceInfo>();
Info = info.Value;
DataContextFactory = dataContextFactory;
MigrationCache = new ScopedCacheClient(_cache, MigrationCacheScope);
AccountStatusCache = new ScopedCacheClient(_cache, AccountStatusCacheScope);
}
public async Task <HealthCheckResult> CheckCacheAsync()
{
var sw = Stopwatch.StartNew();
try {
var cache = new ScopedCacheClient(_configuration.Cache, "health");
var cacheValue = await cache.GetAsync <string>("__PING__").AnyContext();
if (cacheValue.HasValue)
{
return(HealthCheckResult.NotHealthy("Cache Not Working"));
}
} catch (Exception ex) {
return(HealthCheckResult.NotHealthy("Cache Not Working: " + ex.Message));
} finally {
sw.Stop();
_logger.LogTrace("Checking cache took {Duration:g}", sw.Elapsed);
}
return(HealthCheckResult.Healthy);
}
public async Task <HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
{
var sw = Stopwatch.StartNew();
try {
var cache = new ScopedCacheClient(_cache, "health");
var cacheValue = await cache.GetAsync <string>("__PING__").AnyContext();
if (cacheValue.HasValue)
{
return(HealthCheckResult.Unhealthy("Cache Not Working"));
}
} catch (Exception ex) {
return(HealthCheckResult.Unhealthy("Cache Not Working.", ex));
} finally {
sw.Stop();
_logger.LogTrace("Checking cache took {Duration:g}", sw.Elapsed);
}
return(HealthCheckResult.Healthy());
}
public NetInfoJob(IQueue <NetInfoWorkItem> queue, ILoggerFactory loggerFactory, INetInfoService netInfoService,
IHybridCacheClient cacheClient) : base(queue, loggerFactory)
{
_netInfoService = netInfoService;
_statsCache = new ScopedHybridCacheClient(cacheClient, CacheScopes.Stats);
}
private async Task<ReindexResult> InternalReindexAsync(ReindexWorkItem workItem, Func<int, string, Task> progressCallbackAsync, int startProgress = 0, int endProgress = 100, DateTime? startTime = null) {
const string scroll = "5m";
string timestampField = workItem.TimestampField ?? "_timestamp";
var scopedCacheClient = new ScopedCacheClient(_cache, workItem.GetHashCode().ToString());
var settingsResponse = await _client.GetIndexSettingsAsync(s => s.Index(workItem.OldIndex)).AnyContext();
if (!settingsResponse.IsValid)
throw new ApplicationException("Unable to retrieve index settings.");
int scrollSize = 500 / settingsResponse.IndexSettings.NumberOfShards ?? 50;
var scanResults = await _client.SearchAsync<object>(s => s
.Index(workItem.OldIndex)
.AllTypes()
.Query(q => q.Filtered(f => {
if (startTime.HasValue)
f.Filter(f1 => f1.Range(r => r.OnField(timestampField).Greater(startTime.Value)));
}))
.Fields("_source", "_parent")
.Size(scrollSize)
.SearchType(SearchType.Scan)
.Scroll(scroll)).AnyContext();
_logger.Info(scanResults.GetRequest());
if (!scanResults.IsValid || scanResults.ScrollId == null) {
_logger.Error().Exception(scanResults.ConnectionStatus.OriginalException).Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write();
return new ReindexResult();
}
var results = await _client.ScrollAsync<JObject>("5m", scanResults.ScrollId).AnyContext();
if (!results.IsValid) {
await scopedCacheClient.RemoveAsync("id").AnyContext();
return await InternalReindexAsync(workItem, progressCallbackAsync, startProgress, endProgress, startTime).AnyContext();
}
double completed = 0;
long totalHits = results.Total;
while (results.Hits.Any()) {
ISearchResponse<JObject> results1 = results;
IBulkResponse bulkResponse = null;
try {
bulkResponse = await Run.WithRetriesAsync(() => _client.BulkAsync(b => {
foreach (var h in results1.Hits)
ConfigureIndexItem(b, h, workItem.NewIndex);
return b;
}), logger: _logger).AnyContext();
} catch (Exception ex) {
_logger.Error(ex, $"Error trying to do bulk index: {ex.Message}");
}
if (bulkResponse == null || !bulkResponse.IsValid || bulkResponse.ItemsWithErrors.Any()) {
string message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} completed={completed} message={bulkResponse.GetErrorMessage()}";
_logger.Warn(bulkResponse.ConnectionStatus.OriginalException, message);
// try each doc individually so we can see which doc is breaking us
var hitsToRetry = bulkResponse.ItemsWithErrors.Select(i => results.Hits.First(hit => hit.Id == i.Id));
foreach (var itemWithError in hitsToRetry) {
var response = await _client.IndexAsync(itemWithError.Source, d => ConfigureItem(d, itemWithError, workItem.NewIndex)).AnyContext();
if (response.IsValid)
continue;
message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={itemWithError.Id} completed={completed} message={response.GetErrorMessage()}";
_logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message);
var errorDoc = new JObject(new {
itemWithError.Type,
Content = itemWithError.Source.ToString(Formatting.Indented)
});
// put the document into an error index
response = await _client.IndexAsync(errorDoc, d => d.Index(workItem.NewIndex + "-error").Id(itemWithError.Id)).AnyContext();
if (response.IsValid)
continue;
message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={itemWithError.Id} completed={completed} message={response.GetErrorMessage()}";
_logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message);
throw new ReindexException(response.ConnectionStatus, message);
}
}
completed += results.Hits.Count();
await progressCallbackAsync(CalculateProgress(totalHits, (long)completed, startProgress, endProgress), $"Total: {totalHits} Completed: {completed}").AnyContext();
results = await _client.ScrollAsync<JObject>("5m", results.ScrollId).AnyContext();
await scopedCacheClient.AddAsync("id", results.ScrollId, TimeSpan.FromHours(1)).AnyContext();
}
await scopedCacheClient.RemoveAllAsync(new[] { "id" }).AnyContext();
return new ReindexResult { Total = totalHits, Completed = (long)completed };
}
public virtual void CanUseScopedCaches()
{
var cache = GetCacheClient();
if (cache == null)
return;
using (cache) {
var scopedCache1 = new ScopedCacheClient(cache, "scoped1");
var nestedScopedCache1 = new ScopedCacheClient(scopedCache1, "nested");
var scopedCache2 = new ScopedCacheClient(cache, "scoped2");
cache.Set("test", 1);
scopedCache1.Set("test", 2);
nestedScopedCache1.Set("test", 3);
Assert.Equal(1, cache.Get<int>("test"));
Assert.Equal(2, scopedCache1.Get<int>("test"));
Assert.Equal(3, nestedScopedCache1.Get<int>("test"));
Assert.Equal(3, scopedCache1.Get<int>("nested:test"));
Assert.Equal(3, cache.Get<int>("scoped1:nested:test"));
scopedCache2.Set("test", 1);
scopedCache1.FlushAll();
Assert.Null(scopedCache1.Get<int?>("test"));
Assert.Null(nestedScopedCache1.Get<int?>("test"));
Assert.Equal(1, cache.Get<int>("test"));
Assert.Equal(1, scopedCache2.Get<int>("test"));
scopedCache2.FlushAll();
Assert.Null(scopedCache1.Get<int?>("test"));
Assert.Null(nestedScopedCache1.Get<int?>("test"));
Assert.Null(scopedCache2.Get<int?>("test"));
Assert.Equal(1, cache.Get<int>("test"));
}
}
protected void DisableCache()
{
IsCacheEnabled = false;
_scopedCacheClient = new ScopedCacheClient(new NullCacheClient(), EntityTypeName);
}
private void SetCache(ICacheClient cache)
{
IsCacheEnabled = cache != null;
_scopedCacheClient = new ScopedCacheClient(cache ?? new NullCacheClient(), EntityTypeName);
}
private async Task <ReindexResult> InternalReindexAsync(ReindexWorkItem workItem, Func <int, string, Task> progressCallbackAsync, int startProgress = 0, int endProgress = 100, DateTime?startTime = null)
{
const string scroll = "5m";
bool errorIndexCreated = false;
string timestampField = workItem.TimestampField ?? "_timestamp";
var scopedCacheClient = new ScopedCacheClient(_cache, workItem.GetHashCode().ToString());
var settingsResponse = await _client.GetIndexSettingsAsync(s => s.Index(workItem.OldIndex)).AnyContext();
if (!settingsResponse.IsValid)
{
throw new ApplicationException("Unable to retrieve index settings.");
}
int scrollSize = 500 / settingsResponse.IndexSettings.NumberOfShards ?? 50;
var scanResults = await _client.SearchAsync <object>(s => s
.Index(workItem.OldIndex)
.AllTypes()
.Query(q => q.Filtered(f => {
if (startTime.HasValue)
{
f.Filter(f1 => f1.Range(r => r.OnField(timestampField).Greater(startTime.Value)));
}
}))
.Fields("_source", "_parent")
.Size(scrollSize)
.SearchType(SearchType.Scan)
.Scroll(scroll)).AnyContext();
_logger.Info(scanResults.GetRequest());
if (!scanResults.IsValid || scanResults.ScrollId == null)
{
_logger.Error().Exception(scanResults.ConnectionStatus.OriginalException).Message("Invalid search result: message={0}", scanResults.GetErrorMessage()).Write();
return(new ReindexResult());
}
var results = await _client.ScrollAsync <JObject>("5m", scanResults.ScrollId).AnyContext();
if (!results.IsValid)
{
await scopedCacheClient.RemoveAsync("id").AnyContext();
return(await InternalReindexAsync(workItem, progressCallbackAsync, startProgress, endProgress, startTime).AnyContext());
}
double completed = 0;
long totalHits = results.Total;
while (results.Hits.Any())
{
ISearchResponse <JObject> results1 = results;
IBulkResponse bulkResponse = null;
try {
bulkResponse = await Run.WithRetriesAsync(() => _client.BulkAsync(b => {
foreach (var h in results1.Hits)
{
ConfigureIndexItem(b, h, workItem.NewIndex);
}
return(b);
}), logger : _logger, maxAttempts : 2).AnyContext();
} catch (Exception ex) {
_logger.Error(ex, $"Error trying to do bulk index: {ex.Message}");
}
if (bulkResponse == null || !bulkResponse.IsValid || bulkResponse.ItemsWithErrors.Any())
{
string message;
if (bulkResponse != null)
{
message = $"Reindex bulk error: old={workItem.OldIndex} new={workItem.NewIndex} completed={completed} message={bulkResponse?.GetErrorMessage()}";
_logger.Warn(bulkResponse.ConnectionStatus.OriginalException, message);
}
// try each doc individually so we can see which doc is breaking us
var hitsToRetry = bulkResponse?.ItemsWithErrors.Select(i => results1.Hits.First(hit => hit.Id == i.Id)) ?? results1.Hits;
foreach (var itemToRetry in hitsToRetry)
{
IIndexResponse response;
try {
response = await _client.IndexAsync(itemToRetry.Source, d => ConfigureItem(d, itemToRetry, workItem.NewIndex)).AnyContext();
if (response.IsValid)
{
continue;
}
message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={itemToRetry.Id} completed={completed} message={response.GetErrorMessage()}";
_logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message);
var errorDoc = new JObject {
["Type"] = itemToRetry.Type,
["Content"] = itemToRetry.Source.ToString(Formatting.Indented)
};
var errorIndex = workItem.NewIndex + "-error";
if (!errorIndexCreated && !(await _client.IndexExistsAsync(errorIndex).AnyContext()).Exists)
{
await _client.CreateIndexAsync(errorIndex).AnyContext();
errorIndexCreated = true;
}
// put the document into an error index
response = await _client.IndexAsync(errorDoc, d => d.Index(errorIndex).Id(itemToRetry.Id)).AnyContext();
if (response.IsValid)
{
continue;
}
} catch {
throw;
}
message = $"Reindex error: old={workItem.OldIndex} new={workItem.NewIndex} id={itemToRetry.Id} completed={completed} message={response.GetErrorMessage()}";
_logger.Error().Exception(response.ConnectionStatus.OriginalException).Message(message);
throw new ReindexException(response.ConnectionStatus, message);
}
}
completed += results.Hits.Count();
await progressCallbackAsync(CalculateProgress(totalHits, (long)completed, startProgress, endProgress), $"Total: {totalHits} Completed: {completed}").AnyContext();
results = await _client.ScrollAsync <JObject>("5m", results.ScrollId).AnyContext();
await scopedCacheClient.AddAsync("id", results.ScrollId, TimeSpan.FromHours(1)).AnyContext();
}
await scopedCacheClient.RemoveAllAsync(new[] { "id" }).AnyContext();
return(new ReindexResult {
Total = totalHits, Completed = (long)completed
});
}
protected override async Task <JobResult> ProcessQueueEntryAsync(QueueEntryContext <WebHookNotification> context)
{
var body = context.QueueEntry.Value;
bool shouldLog = body.ProjectId != _appOptions.Value.InternalProjectId;
using (_logger.BeginScope(new ExceptionlessState().Organization(body.OrganizationId).Project(body.ProjectId))) {
if (shouldLog)
{
_logger.LogTrace("Process web hook call: id={Id} project={1} url={Url}", context.QueueEntry.Id, body.ProjectId, body.Url);
}
if (!await IsEnabledAsync(body).AnyContext())
{
_logger.LogInformation("Web hook cancelled: Web hook is disabled");
return(JobResult.Cancelled);
}
var cache = new ScopedCacheClient(_cacheClient, GetCacheKeyScope(body));
long consecutiveErrors = await cache.GetAsync <long>(ConsecutiveErrorsCacheKey, 0).AnyContext();
if (consecutiveErrors > 10)
{
var lastAttempt = await cache.GetAsync(LastAttemptCacheKey, SystemClock.UtcNow).AnyContext();
var nextAttemptAllowedAt = lastAttempt.AddMinutes(15);
if (nextAttemptAllowedAt >= SystemClock.UtcNow)
{
_logger.LogInformation("Web hook cancelled due to {FailureCount} consecutive failed attempts. Will be allowed to try again at {NextAttempt}.", consecutiveErrors, nextAttemptAllowedAt);
return(JobResult.Cancelled);
}
}
bool successful = true;
HttpResponseMessage response = null;
try {
using (var timeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5))) {
using (var postCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, timeoutCancellationTokenSource.Token)) {
response = await _client.PostAsJsonAsync(body.Url, body.Data.ToJson(Formatting.Indented, _jsonSerializerSettings), postCancellationTokenSource.Token).AnyContext();
if (!response.IsSuccessStatusCode)
{
successful = false;
}
else if (consecutiveErrors > 0)
{
await cache.RemoveAllAsync(_cacheKeys).AnyContext();
}
}
}
} catch (OperationCanceledException ex) {
successful = false;
if (shouldLog)
{
_logger.LogError(ex, "Timeout calling web hook: status={Status} org={organization} project={project} url={Url}", response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url);
}
return(JobResult.Cancelled);
} catch (Exception ex) {
successful = false;
if (shouldLog)
{
_logger.LogError(ex, "Error calling web hook: status={Status} org={organization} project={project} url={Url}", response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url);
}
return(JobResult.FromException(ex));
} finally {
if (successful)
{
_logger.LogInformation("Web hook POST complete: status={Status} org={organization} project={project} url={Url}", response?.StatusCode, body.OrganizationId, body.ProjectId, body.Url);
}
else if (response != null && (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.Gone))
{
_logger.LogWarning("Disabling Web hook instance {WebHookId} due to status code: status={Status} org={organization} project={project} url={Url}", body.Type == WebHookType.Slack ? "Slack" : body.WebHookId, response.StatusCode, body.OrganizationId, body.ProjectId, body.Url);
await DisableIntegrationAsync(body).AnyContext();
await cache.RemoveAllAsync(_cacheKeys).AnyContext();
}
else
{
var now = SystemClock.UtcNow;
await cache.SetAsync(LastAttemptCacheKey, now, TimeSpan.FromDays(3)).AnyContext();
consecutiveErrors = await cache.IncrementAsync(ConsecutiveErrorsCacheKey, TimeSpan.FromDays(3)).AnyContext();
DateTime firstAttempt;
if (consecutiveErrors == 1)
{
await cache.SetAsync(FirstAttemptCacheKey, now, TimeSpan.FromDays(3)).AnyContext();
firstAttempt = now;
}
else
{
firstAttempt = await cache.GetAsync(FirstAttemptCacheKey, now).AnyContext();
}
if (consecutiveErrors >= 10)
{
// don't retry any more
context.QueueEntry.MarkCompleted();
// disable if more than 10 consecutive errors over the course of multiple days
if (firstAttempt.IsBefore(now.SubtractDays(2)))
{
_logger.LogWarning("Disabling Web hook instance {WebHookId} due to too many consecutive failures.", body.Type == WebHookType.Slack ? "Slack" : body.WebHookId);
await DisableIntegrationAsync(body).AnyContext();
await cache.RemoveAllAsync(_cacheKeys).AnyContext();
}
}
}
}
}
return(JobResult.Success);
}
protected override async Task <JobResult> RunInternalAsync(JobContext context)
{
var elasticOptions = _configuration.Options;
if (elasticOptions.ElasticsearchToMigrate == null)
{
return(JobResult.CancelledWithMessage($"Please configure the connection string EX_{nameof(elasticOptions.ElasticsearchToMigrate)}."));
}
var retentionPeriod = _configuration.Events.MaxIndexAge.GetValueOrDefault(TimeSpan.FromDays(180));
string sourceScope = elasticOptions.ElasticsearchToMigrate.Scope;
string scope = elasticOptions.ScopePrefix;
var cutOffDate = elasticOptions.ReindexCutOffDate;
var client = _configuration.Client;
await _configuration.ConfigureIndexesAsync().AnyContext();
var workItemQueue = new Queue <ReindexWorkItem>();
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}organizations-v1", "organization", $"{scope}organizations-v1", "updated_utc"));
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}organizations-v1", "project", $"{scope}projects-v1", "updated_utc"));
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}organizations-v1", "token", $"{scope}tokens-v1", "updated_utc"));
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}organizations-v1", "user", $"{scope}users-v1", "updated_utc"));
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}organizations-v1", "webhook", $"{scope}webhooks-v1", "created_utc", script: MIGRATE_VERSION_SCRIPT));
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}stacks-v1", "stacks", $"{scope}stacks-v1", "last_occurrence"));
// create the new indexes, don't migrate yet
foreach (var index in _configuration.Indexes.OfType <DailyIndex>())
{
for (int day = 0; day <= retentionPeriod.Days; day++)
{
var date = day == 0 ? SystemClock.UtcNow : SystemClock.UtcNow.SubtractDays(day);
string indexToCreate = $"{scope}events-v1-{date:yyyy.MM.dd}";
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}events-v1-{date:yyyy.MM.dd}", "events", indexToCreate, "updated_utc", () => index.EnsureIndexAsync(date)));
}
}
// Reset the alias cache
var aliasCache = new ScopedCacheClient(_configuration.Cache, "alias");
await aliasCache.RemoveAllAsync().AnyContext();
var started = SystemClock.UtcNow;
var lastProgress = SystemClock.UtcNow;
int retriesCount = 0;
int totalTasks = workItemQueue.Count;
var workingTasks = new List <ReindexWorkItem>();
var completedTasks = new List <ReindexWorkItem>();
var failedTasks = new List <ReindexWorkItem>();
while (true)
{
if (workingTasks.Count == 0 && workItemQueue.Count == 0)
{
break;
}
if (workingTasks.Count < 10 && workItemQueue.TryDequeue(out var dequeuedWorkItem))
{
if (dequeuedWorkItem.CreateIndex != null)
{
try {
await dequeuedWorkItem.CreateIndex().AnyContext();
} catch (Exception ex) {
_logger.LogError(ex, "Failed to create index for {TargetIndex}", dequeuedWorkItem.TargetIndex);
continue;
}
}
int batchSize = 1000;
if (dequeuedWorkItem.Attempts == 1)
{
batchSize = 500;
}
else if (dequeuedWorkItem.Attempts >= 2)
{
batchSize = 250;
}
var response = await client.ReindexOnServerAsync(r => r
.Source(s => s
.Remote(ConfigureRemoteElasticSource)
.Index(dequeuedWorkItem.SourceIndex)
.Size(batchSize)
.Query <object>(q => {
var container = q.Term("_type", dequeuedWorkItem.SourceIndexType);
if (!String.IsNullOrEmpty(dequeuedWorkItem.DateField))
{
container &= q.DateRange(d => d.Field(dequeuedWorkItem.DateField).GreaterThanOrEquals(cutOffDate));
}
return(container);
}))
.Destination(d => d
.Index(dequeuedWorkItem.TargetIndex))
.Conflicts(Conflicts.Proceed)
.WaitForCompletion(false)
.Script(s => {
if (!String.IsNullOrEmpty(dequeuedWorkItem.Script))
{
return(s.Source(dequeuedWorkItem.Script));
}
return(null);
})).AnyContext();
dequeuedWorkItem.Attempts += 1;
dequeuedWorkItem.TaskId = response.Task;
workingTasks.Add(dequeuedWorkItem);
_logger.LogInformation("STARTED - {TargetIndex} A:{Attempts} ({TaskId})...", dequeuedWorkItem.TargetIndex, dequeuedWorkItem.Attempts, dequeuedWorkItem.TaskId);
continue;
}
double highestProgress = 0;
foreach (var workItem in workingTasks.ToArray())
{
var taskStatus = await client.Tasks.GetTaskAsync(workItem.TaskId, t => t.WaitForCompletion(false)).AnyContext();
_logger.LogRequest(taskStatus);
var status = taskStatus?.Task?.Status;
if (status == null)
{
_logger.LogWarning(taskStatus?.OriginalException, "Error getting task status for {TargetIndex} {TaskId}: {Message}", workItem.TargetIndex, workItem.TaskId, taskStatus.GetErrorMessage());
if (taskStatus?.ServerError?.Status == 429)
{
await Task.Delay(TimeSpan.FromSeconds(1));
}
continue;
}
var duration = TimeSpan.FromMilliseconds(taskStatus.Task.RunningTimeInNanoseconds * 0.000001);
double progress = status.Total > 0 ? (status.Created + status.Updated + status.Deleted + status.VersionConflicts * 1.0) / status.Total : 0;
highestProgress = Math.Max(highestProgress, progress);
if (!taskStatus.IsValid)
{
_logger.LogWarning(taskStatus.OriginalException, "Error getting task status for {TargetIndex} ({TaskId}): {Message}", workItem.TargetIndex, workItem.TaskId, taskStatus.GetErrorMessage());
workItem.ConsecutiveStatusErrors++;
if (taskStatus.Completed || workItem.ConsecutiveStatusErrors > 5)
{
workingTasks.Remove(workItem);
workItem.LastTaskInfo = taskStatus.Task;
if (taskStatus.Completed && workItem.Attempts < 3)
{
_logger.LogWarning("FAILED RETRY - {TargetIndex} in {Duration:hh\\:mm} C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", workItem.TargetIndex, duration, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, workItem.Attempts, workItem.TaskId);
workItem.ConsecutiveStatusErrors = 0;
workItemQueue.Enqueue(workItem);
totalTasks++;
retriesCount++;
await Task.Delay(TimeSpan.FromSeconds(15)).AnyContext();
}
else
{
_logger.LogCritical("FAILED - {TargetIndex} in {Duration:hh\\:mm} C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", workItem.TargetIndex, duration, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, workItem.Attempts, workItem.TaskId);
failedTasks.Add(workItem);
}
}
continue;
}
if (!taskStatus.Completed)
{
continue;
}
workingTasks.Remove(workItem);
workItem.LastTaskInfo = taskStatus.Task;
completedTasks.Add(workItem);
var targetCount = await client.CountAsync <object>(d => d.Index(workItem.TargetIndex)).AnyContext();
_logger.LogInformation("COMPLETED - {TargetIndex} ({TargetCount}) in {Duration:hh\\:mm} C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", workItem.TargetIndex, targetCount.Count, duration, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, workItem.Attempts, workItem.TaskId);
}
if (SystemClock.UtcNow.Subtract(lastProgress) > TimeSpan.FromMinutes(5))
{
_logger.LogInformation("STATUS - I:{Completed}/{Total} P:{Progress:F0}% T:{Duration:d\\.hh\\:mm} W:{Working} F:{Failed} R:{Retries}", completedTasks.Count, totalTasks, highestProgress * 100, SystemClock.UtcNow.Subtract(started), workingTasks.Count, failedTasks.Count, retriesCount);
lastProgress = SystemClock.UtcNow;
}
await Task.Delay(TimeSpan.FromSeconds(2));
}
_logger.LogInformation("----- REINDEX COMPLETE", completedTasks.Count, totalTasks, SystemClock.UtcNow.Subtract(started), failedTasks.Count, retriesCount);
foreach (var task in completedTasks)
{
var status = task.LastTaskInfo.Status;
var duration = TimeSpan.FromMilliseconds(task.LastTaskInfo.RunningTimeInNanoseconds * 0.000001);
double progress = status.Total > 0 ? (status.Created + status.Updated + status.Deleted + status.VersionConflicts * 1.0) / status.Total : 0;
var targetCount = await client.CountAsync <object>(d => d.Index(task.TargetIndex)).AnyContext();
_logger.LogInformation("SUCCESS - {TargetIndex} ({TargetCount}) in {Duration:hh\\:mm} C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", task.TargetIndex, targetCount.Count, duration, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, task.Attempts, task.TaskId);
}
foreach (var task in failedTasks)
{
var status = task.LastTaskInfo.Status;
var duration = TimeSpan.FromMilliseconds(task.LastTaskInfo.RunningTimeInNanoseconds * 0.000001);
double progress = status.Total > 0 ? (status.Created + status.Updated + status.Deleted + status.VersionConflicts * 1.0) / status.Total : 0;
var targetCount = await client.CountAsync <object>(d => d.Index(task.TargetIndex));
_logger.LogCritical("FAILED - {TargetIndex} ({TargetCount}) in {Duration:hh\\:mm} C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", task.TargetIndex, targetCount.Count, duration, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, task.Attempts, task.TaskId);
}
_logger.LogInformation("----- SUMMARY - I:{Completed}/{Total} T:{Duration:d\\.hh\\:mm} F:{Failed} R:{Retries}", completedTasks.Count, totalTasks, SystemClock.UtcNow.Subtract(started), failedTasks.Count, retriesCount);
_logger.LogInformation("Updating aliases");
await _configuration.MaintainIndexesAsync();
_logger.LogInformation("Updated aliases");
return(JobResult.Success);
}
