public async Task ProcessItemsAsync()
{
IList <Task> indexTaskList = new List <Task>();
IList <SearchIndex> searchIndices = await this._kpiService.GetSearchIndicesAsync();
foreach (SearchIndex searchIndex in searchIndices)
{
Task indexPushOperation = ProcessIndexAsync(searchIndex);
indexTaskList.Add(indexPushOperation);
}
await Task.WhenAll(indexTaskList);
ConsoleLogging.LogLine($"KPI Process done for {searchIndices.Count} indices", ConsoleLogging.LogSeverity.Info);
}
public Task LogProcessAsync()
{
ConsoleLogging.LogLine("{{{PULSE}}} Log Queue Process", severity: ConsoleLogging.LogSeverity.Info);
if (notIsLoggingLocked)
{
notIsLoggingLocked = false;
CommonFunctions.LogManager.ProcessLogQueue();
notIsLoggingLocked = true;
}
else
{
ConsoleLogging.LogLine("[[[BYPASS]]] Log Queue Process is locked and in progress!", severity: ConsoleLogging.LogSeverity.Warning);
}
return(Task.CompletedTask);
}
public async Task KPIProcessAsync()
{
ConsoleLogging.LogLine("{{{PULSE}}} KPI Process", severity: ConsoleLogging.LogSeverity.Info);
if (notIsProcessLocked)
{
notIsProcessLocked = false;
ServiceManager.Initialize();
await context.ProcessItemsAsync();
notIsProcessLocked = true;
}
else
{
ConsoleLogging.LogLine("[[[BYPASS]]] KPI Process is locked and in progress!", severity: ConsoleLogging.LogSeverity.Warning);
}
}
private async Task ProcessIndexAsync(SearchIndex searchIndex)
{
DateTime startDate = this._kpiService.GetSearchRange(searchIndex.IndexId); // Get last log insertion date
int fragmentCount = CalculateLoopCount(startDate); // Calculate fragment count
DateTime searchRange = startDate;
for (int i = 0; i < fragmentCount; i++)
{
searchRange = searchRange.AddMinutes(CommonFunctions.UnifyingConstant); // add 15 min for each iteration.
string requestBody = ElasticSearchRESTAdapter.GetRequestBody(start: searchRange);
string fullIndexName = ElasticSearchRESTAdapter.GetFullIndexName(index: searchIndex.IndexName, indexPattern: searchIndex.IndexPattern, indexPatternValue: searchRange);
Root responseRoot = await ElasticSearchRESTAdapter.GetResponseFromElasticUrlAsync(urlAddress : searchIndex.UrlAddress, index : fullIndexName, requestBody : requestBody);
if (responseRoot == null)
{
// Root is null for current search range
// Check for next day of search range > today
// Example: searchRange: 05.01.2020 -> 06.01.2020 > 10.01.2020 ?
double checkForNextDay = (DateTime.Now - searchRange.AddDays(1)).TotalMilliseconds;
if (checkForNextDay < 0)
{
break; // it became greater than current date.
}
searchRange = searchRange.AddDays(1); // Increase search range by one day
// then increase the current loop iterator for increment above to prevent out of range exception.
// 24 Hour * 60 Min / fragmentation by minute) - 1 for current iteration
i += (24 * 60 / CommonFunctions.UnifyingConstant) - 1;
ConsoleLogging.LogLine($"{searchIndex.UrlAddress}-> {searchIndex.IndexName} is expired, skipped to [{i + 1}/{fragmentCount}].");
continue;
}
InsertDataToDatabase(aggregation: responseRoot.Aggregation, indexId: searchIndex.IndexId, logDate: searchRange);
ConsoleLogging.LogLine($"{searchIndex.UrlAddress}-> {searchIndex.IndexName} [{i + 1}/{fragmentCount}] added.");
}
}