public async Task <IHttpActionResult> RunJobAsync(string name)
{
switch (name.ToLowerInvariant())
{
case "indexes":
await _configuration.ConfigureIndexesAsync(beginReindexingOutdated : false);
break;
case "update-organization-plans":
await _workItemQueue.EnqueueAsync(new OrganizationMaintenanceWorkItem { UpgradePlans = true });
break;
case "update-project-default-bot-lists":
await _workItemQueue.EnqueueAsync(new ProjectMaintenanceWorkItem { UpdateDefaultBotList = true, IncrementConfigurationVersion = true });
break;
case "increment-project-configuration-version":
await _workItemQueue.EnqueueAsync(new ProjectMaintenanceWorkItem { IncrementConfigurationVersion = true });
break;
default:
return(NotFound());
}
return(Ok());
}
public IntegrationTestsBase(ITestOutputHelper output) : base(output)
{
var builder = new MvcWebApplicationBuilder <Startup>()
.UseSolutionRelativeContentRoot("src/Exceptionless.Api")
.ConfigureBeforeStartup(Configure)
.ConfigureAfterStartup(RegisterServices)
.UseApplicationAssemblies();
_server = builder.Build();
var settings = GetService <Newtonsoft.Json.JsonSerializerSettings>();
_serializer = GetService <ITextSerializer>();
_client = new FluentClient(new JsonContentSerializer(settings), _server.CreateHandler())
{
BaseUri = new Uri(_server.BaseAddress + "api/v2")
};
_httpClient = new HttpClient(_server.CreateHandler())
{
BaseAddress = new Uri(_server.BaseAddress + "api/v2/")
};
_configuration = GetService <ExceptionlessElasticConfiguration>();
_configuration.DeleteIndexesAsync().GetAwaiter().GetResult();
_configuration.ConfigureIndexesAsync(beginReindexingOutdated: false).GetAwaiter().GetResult();
}
protected override async Task <JobResult> RunInternalAsync(JobContext context)
{
await _configuration.ConfigureIndexesAsync(null, false).AnyContext();
await _migrationManager.RunMigrationsAsync().AnyContext();
var tasks = _configuration.Indexes.OfType <VersionedIndex>().Select(ReindexIfNecessary);
await Task.WhenAll(tasks).AnyContext();
return(JobResult.Success);
}
protected virtual async Task ResetDataAsync()
{
await _semaphoreSlim.WaitAsync();
try {
var oldLoggingLevel = Log.MinimumLevel;
Log.MinimumLevel = LogLevel.Warning;
await RefreshDataAsync();
if (!_indexesHaveBeenConfigured)
{
await _configuration.DeleteIndexesAsync();
await _configuration.ConfigureIndexesAsync();
_indexesHaveBeenConfigured = true;
}
else
{
string indexes = String.Join(',', _configuration.Indexes.Select(i => i.Name));
await _configuration.Client.DeleteByQueryAsync(new DeleteByQueryRequest(indexes) {
Query = new MatchAllQuery(),
IgnoreUnavailable = true,
Refresh = true
});
}
_logger.LogTrace("Configured Indexes");
foreach (var index in _configuration.Indexes)
{
index.QueryParser.Configuration.MappingResolver.RefreshMapping();
}
var cacheClient = GetService <ICacheClient>();
await cacheClient.RemoveAllAsync();
var fileStorage = GetService <IFileStorage>();
await fileStorage.DeleteFilesAsync(await fileStorage.GetFileListAsync());
await GetService <IQueue <WorkItemData> >().DeleteQueueAsync();
Log.MinimumLevel = oldLoggingLevel;
} finally {
_semaphoreSlim.Release();
_logger.LogDebug("Reset Data");
}
}
public async Task <IActionResult> RunJobAsync(string name)
{
switch (name.ToLowerInvariant())
{
case "indexes":
if (!Settings.Current.DisableIndexConfiguration)
{
await _configuration.ConfigureIndexesAsync(beginReindexingOutdated : false);
}
break;
case "update-organization-plans":
await _workItemQueue.EnqueueAsync(new OrganizationMaintenanceWorkItem { UpgradePlans = true });
break;
case "remove-old-organization-usage":
await _workItemQueue.EnqueueAsync(new OrganizationMaintenanceWorkItem { RemoveOldUsageStats = true });
break;
case "update-project-default-bot-lists":
await _workItemQueue.EnqueueAsync(new ProjectMaintenanceWorkItem { UpdateDefaultBotList = true, IncrementConfigurationVersion = true });
break;
case "increment-project-configuration-version":
await _workItemQueue.EnqueueAsync(new ProjectMaintenanceWorkItem { IncrementConfigurationVersion = true });
break;
case "remove-old-project-usage":
await _workItemQueue.EnqueueAsync(new ProjectMaintenanceWorkItem { RemoveOldUsageStats = true });
break;
case "normalize-user-email-address":
await _workItemQueue.EnqueueAsync(new UserMaintenanceWorkItem { Normalize = true });
break;
default:
return(NotFound());
}
return(Ok());
}
protected virtual async Task ResetDataAsync()
{
await _semaphoreSlim.WaitAsync();
try {
var oldLoggingLevel = Log.MinimumLevel;
Log.MinimumLevel = LogLevel.Warning;
bool isTraceLogLevelEnabled = _logger.IsEnabled(LogLevel.Trace);
await _configuration.DeleteIndexesAsync();
await _configuration.ConfigureIndexesAsync();
if (isTraceLogLevelEnabled)
{
_logger.LogTrace("Configured Indexes");
}
foreach (var index in _configuration.Indexes)
{
index.QueryParser.Configuration.RefreshMapping();
}
var cacheClient = GetService <ICacheClient>();
await cacheClient.RemoveAllAsync();
var fileStorage = GetService <IFileStorage>();
await fileStorage.DeleteFilesAsync(await fileStorage.GetFileListAsync());
await GetService <IQueue <WorkItemData> >().DeleteQueueAsync();
Log.MinimumLevel = oldLoggingLevel;
} finally {
_semaphoreSlim.Release();
}
}
public ElasticTestBase(ITestOutputHelper output) : base(output)
{
_configuration = GetService <ExceptionlessElasticConfiguration>();
_configuration.DeleteIndexesAsync().GetAwaiter().GetResult();
_configuration.ConfigureIndexesAsync(beginReindexingOutdated: false).GetAwaiter().GetResult();
}
protected virtual async Task ResetDataAsync()
{
await _semaphoreSlim.WaitAsync();
try {
var oldLoggingLevel = Log.MinimumLevel;
Log.MinimumLevel = LogLevel.Warning;
bool isTraceLogLevelEnabled = _logger.IsEnabled(LogLevel.Trace);
string indexList = String.Join(',', _configuration.Indexes.Select(i => i.Name));
await _configuration.Client.RefreshAsync(indexList);
if (!_indexesHaveBeenConfigured)
{
await _configuration.DeleteIndexesAsync();
await _configuration.ConfigureIndexesAsync();
_indexesHaveBeenConfigured = true;
if (isTraceLogLevelEnabled)
{
_logger.LogTrace("Configured Indexes");
}
}
else
{
var response = await _configuration.Client.DeleteByQueryAsync(new DeleteByQueryRequest(indexList) {
Query = new MatchAllQuery(),
IgnoreUnavailable = true,
WaitForCompletion = true,
Refresh = true
});
if (isTraceLogLevelEnabled)
{
_logger.LogTraceRequest(response);
}
}
foreach (var index in _configuration.Indexes)
{
foreach (var type in index.IndexTypes)
{
type.QueryParser.Configuration.RefreshMapping();
}
}
var cacheClient = GetService <ICacheClient>();
await cacheClient.RemoveAllAsync();
var fileStorage = GetService <IFileStorage>();
await fileStorage.DeleteFilesAsync(await fileStorage.GetFileListAsync());
await GetService <IQueue <WorkItemData> >().DeleteQueueAsync();
Log.MinimumLevel = oldLoggingLevel;
} finally {
_semaphoreSlim.Release();
}
}
protected override async Task <JobResult> RunInternalAsync(JobContext context)
{
var elasticOptions = _configuration.Options;
if (elasticOptions.ElasticsearchToMigrate == null)
{
return(JobResult.CancelledWithMessage($"Please configure the connection string EX_{nameof(elasticOptions.ElasticsearchToMigrate)}."));
}
var retentionPeriod = _configuration.Events.MaxIndexAge.GetValueOrDefault(TimeSpan.FromDays(180));
string sourceScope = elasticOptions.ElasticsearchToMigrate.Scope;
string scope = elasticOptions.ScopePrefix;
var cutOffDate = elasticOptions.ReindexCutOffDate;
var client = _configuration.Client;
await _configuration.ConfigureIndexesAsync().AnyContext();
var workItemQueue = new Queue <ReindexWorkItem>();
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}organizations-v1", "organization", $"{scope}organizations-v1", "updated_utc"));
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}organizations-v1", "project", $"{scope}projects-v1", "updated_utc"));
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}organizations-v1", "token", $"{scope}tokens-v1", "updated_utc"));
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}organizations-v1", "user", $"{scope}users-v1", "updated_utc"));
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}organizations-v1", "webhook", $"{scope}webhooks-v1", "created_utc", script: MIGRATE_VERSION_SCRIPT));
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}stacks-v1", "stacks", $"{scope}stacks-v1", "last_occurrence"));
// create the new indexes, don't migrate yet
foreach (var index in _configuration.Indexes.OfType <DailyIndex>())
{
for (int day = 0; day <= retentionPeriod.Days; day++)
{
var date = day == 0 ? SystemClock.UtcNow : SystemClock.UtcNow.SubtractDays(day);
string indexToCreate = $"{scope}events-v1-{date:yyyy.MM.dd}";
workItemQueue.Enqueue(new ReindexWorkItem($"{sourceScope}events-v1-{date:yyyy.MM.dd}", "events", indexToCreate, "updated_utc", () => index.EnsureIndexAsync(date)));
}
}
// Reset the alias cache
var aliasCache = new ScopedCacheClient(_configuration.Cache, "alias");
await aliasCache.RemoveAllAsync().AnyContext();
var started = SystemClock.UtcNow;
var lastProgress = SystemClock.UtcNow;
int retriesCount = 0;
int totalTasks = workItemQueue.Count;
var workingTasks = new List <ReindexWorkItem>();
var completedTasks = new List <ReindexWorkItem>();
var failedTasks = new List <ReindexWorkItem>();
while (true)
{
if (workingTasks.Count == 0 && workItemQueue.Count == 0)
{
break;
}
if (workingTasks.Count < 10 && workItemQueue.TryDequeue(out var dequeuedWorkItem))
{
if (dequeuedWorkItem.CreateIndex != null)
{
try {
await dequeuedWorkItem.CreateIndex().AnyContext();
} catch (Exception ex) {
_logger.LogError(ex, "Failed to create index for {TargetIndex}", dequeuedWorkItem.TargetIndex);
continue;
}
}
int batchSize = 1000;
if (dequeuedWorkItem.Attempts == 1)
{
batchSize = 500;
}
else if (dequeuedWorkItem.Attempts >= 2)
{
batchSize = 250;
}
var response = await client.ReindexOnServerAsync(r => r
.Source(s => s
.Remote(ConfigureRemoteElasticSource)
.Index(dequeuedWorkItem.SourceIndex)
.Size(batchSize)
.Query <object>(q => {
var container = q.Term("_type", dequeuedWorkItem.SourceIndexType);
if (!String.IsNullOrEmpty(dequeuedWorkItem.DateField))
{
container &= q.DateRange(d => d.Field(dequeuedWorkItem.DateField).GreaterThanOrEquals(cutOffDate));
}
return(container);
}))
.Destination(d => d
.Index(dequeuedWorkItem.TargetIndex))
.Conflicts(Conflicts.Proceed)
.WaitForCompletion(false)
.Script(s => {
if (!String.IsNullOrEmpty(dequeuedWorkItem.Script))
{
return(s.Source(dequeuedWorkItem.Script));
}
return(null);
})).AnyContext();
dequeuedWorkItem.Attempts += 1;
dequeuedWorkItem.TaskId = response.Task;
workingTasks.Add(dequeuedWorkItem);
_logger.LogInformation("STARTED - {TargetIndex} A:{Attempts} ({TaskId})...", dequeuedWorkItem.TargetIndex, dequeuedWorkItem.Attempts, dequeuedWorkItem.TaskId);
continue;
}
double highestProgress = 0;
foreach (var workItem in workingTasks.ToArray())
{
var taskStatus = await client.Tasks.GetTaskAsync(workItem.TaskId, t => t.WaitForCompletion(false)).AnyContext();
_logger.LogRequest(taskStatus);
var status = taskStatus?.Task?.Status;
if (status == null)
{
_logger.LogWarning(taskStatus?.OriginalException, "Error getting task status for {TargetIndex} {TaskId}: {Message}", workItem.TargetIndex, workItem.TaskId, taskStatus.GetErrorMessage());
if (taskStatus?.ServerError?.Status == 429)
{
await Task.Delay(TimeSpan.FromSeconds(1));
}
continue;
}
var duration = TimeSpan.FromMilliseconds(taskStatus.Task.RunningTimeInNanoseconds * 0.000001);
double progress = status.Total > 0 ? (status.Created + status.Updated + status.Deleted + status.VersionConflicts * 1.0) / status.Total : 0;
highestProgress = Math.Max(highestProgress, progress);
if (!taskStatus.IsValid)
{
_logger.LogWarning(taskStatus.OriginalException, "Error getting task status for {TargetIndex} ({TaskId}): {Message}", workItem.TargetIndex, workItem.TaskId, taskStatus.GetErrorMessage());
workItem.ConsecutiveStatusErrors++;
if (taskStatus.Completed || workItem.ConsecutiveStatusErrors > 5)
{
workingTasks.Remove(workItem);
workItem.LastTaskInfo = taskStatus.Task;
if (taskStatus.Completed && workItem.Attempts < 3)
{
_logger.LogWarning("FAILED RETRY - {TargetIndex} in {Duration:hh\\:mm} C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", workItem.TargetIndex, duration, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, workItem.Attempts, workItem.TaskId);
workItem.ConsecutiveStatusErrors = 0;
workItemQueue.Enqueue(workItem);
totalTasks++;
retriesCount++;
await Task.Delay(TimeSpan.FromSeconds(15)).AnyContext();
}
else
{
_logger.LogCritical("FAILED - {TargetIndex} in {Duration:hh\\:mm} C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", workItem.TargetIndex, duration, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, workItem.Attempts, workItem.TaskId);
failedTasks.Add(workItem);
}
}
continue;
}
if (!taskStatus.Completed)
{
continue;
}
workingTasks.Remove(workItem);
workItem.LastTaskInfo = taskStatus.Task;
completedTasks.Add(workItem);
var targetCount = await client.CountAsync <object>(d => d.Index(workItem.TargetIndex)).AnyContext();
_logger.LogInformation("COMPLETED - {TargetIndex} ({TargetCount}) in {Duration:hh\\:mm} C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", workItem.TargetIndex, targetCount.Count, duration, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, workItem.Attempts, workItem.TaskId);
}
if (SystemClock.UtcNow.Subtract(lastProgress) > TimeSpan.FromMinutes(5))
{
_logger.LogInformation("STATUS - I:{Completed}/{Total} P:{Progress:F0}% T:{Duration:d\\.hh\\:mm} W:{Working} F:{Failed} R:{Retries}", completedTasks.Count, totalTasks, highestProgress * 100, SystemClock.UtcNow.Subtract(started), workingTasks.Count, failedTasks.Count, retriesCount);
lastProgress = SystemClock.UtcNow;
}
await Task.Delay(TimeSpan.FromSeconds(2));
}
_logger.LogInformation("----- REINDEX COMPLETE", completedTasks.Count, totalTasks, SystemClock.UtcNow.Subtract(started), failedTasks.Count, retriesCount);
foreach (var task in completedTasks)
{
var status = task.LastTaskInfo.Status;
var duration = TimeSpan.FromMilliseconds(task.LastTaskInfo.RunningTimeInNanoseconds * 0.000001);
double progress = status.Total > 0 ? (status.Created + status.Updated + status.Deleted + status.VersionConflicts * 1.0) / status.Total : 0;
var targetCount = await client.CountAsync <object>(d => d.Index(task.TargetIndex)).AnyContext();
_logger.LogInformation("SUCCESS - {TargetIndex} ({TargetCount}) in {Duration:hh\\:mm} C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", task.TargetIndex, targetCount.Count, duration, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, task.Attempts, task.TaskId);
}
foreach (var task in failedTasks)
{
var status = task.LastTaskInfo.Status;
var duration = TimeSpan.FromMilliseconds(task.LastTaskInfo.RunningTimeInNanoseconds * 0.000001);
double progress = status.Total > 0 ? (status.Created + status.Updated + status.Deleted + status.VersionConflicts * 1.0) / status.Total : 0;
var targetCount = await client.CountAsync <object>(d => d.Index(task.TargetIndex));
_logger.LogCritical("FAILED - {TargetIndex} ({TargetCount}) in {Duration:hh\\:mm} C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", task.TargetIndex, targetCount.Count, duration, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, task.Attempts, task.TaskId);
}
_logger.LogInformation("----- SUMMARY - I:{Completed}/{Total} T:{Duration:d\\.hh\\:mm} F:{Failed} R:{Retries}", completedTasks.Count, totalTasks, SystemClock.UtcNow.Subtract(started), failedTasks.Count, retriesCount);
_logger.LogInformation("Updating aliases");
await _configuration.MaintainIndexesAsync();
_logger.LogInformation("Updated aliases");
return(JobResult.Success);
}