public async Task GarbageCollectDataAsync(DataCollectionConfig dataCollectionConfig, IEnumerable <string> newFileNames, IEnumerable <IFileInfo> collectionFileInfos, CancellationToken cancellationToken)
{
try
{
await _dataCollector.GarbageCollectDataAsync(dataCollectionConfig, newFileNames, collectionFileInfos, cancellationToken);
}
catch (Exception e)
{
_logger.LogCritical("Error garbage collecting data '{dataCollectionName}': {errorMessage}", dataCollectionConfig.DataCollectionName, e.GetAggregateMessages());
throw;
}
}
public async Task <bool> PrepareDataAsync(DataCollectionConfig dataCollectionConfig, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(dataCollectionConfig.PrepareUrl))
{
return(false);
}
await PrepareCollectAsync(dataCollectionConfig.PrepareUrl, dataCollectionConfig.PrepareHeaders, dataCollectionConfig.PrepareHttpMethod, cancellationToken);
if (!string.IsNullOrEmpty(dataCollectionConfig.PrepareFinishedPollUrl))
{
await WaitPrepareToFinishAsync(dataCollectionConfig.PrepareFinishedPollUrl, dataCollectionConfig.PrepareFinishedPollHeaders, dataCollectionConfig.PrepareFinishedPollInterval, dataCollectionConfig.PrepareDuration, cancellationToken);
}
else
{
await WaitPrepareDurationAsync(dataCollectionConfig, cancellationToken);
}
return(true);
}
public async Task <bool> PrepareDataAsync(DataCollectionConfig dataCollectionConfig, CancellationToken cancellationToken)
{
try
{
if (!string.IsNullOrEmpty(dataCollectionConfig.PrepareUrl))
{
_logger.LogInformation("Started preparing data '{dataCollectionName}'", dataCollectionConfig.DataCollectionName);
}
var result = await _dataPreparer.PrepareDataAsync(dataCollectionConfig, cancellationToken);
if (!string.IsNullOrEmpty(dataCollectionConfig.PrepareUrl))
{
_logger.LogInformation("Finished preparing data '{dataCollectionName}'", dataCollectionConfig.DataCollectionName);
}
return(result);
}
catch (Exception e)
{
_logger.LogCritical("Error preparing data '{dataCollectionName}': {errorMessage}", dataCollectionConfig.DataCollectionName, e.GetAggregateMessages());
throw;
}
}
private async Task WaitPrepareDurationAsync(DataCollectionConfig dataCollectionConfig, CancellationToken cancellationToken)
{
await _delay.DelayAsync(dataCollectionConfig.PrepareDuration ?? default, $"Data {dataCollectionConfig.DataCollectionName} prepare to finish", cancellationToken);
}
public async Task <(IEnumerable <string> NewFileNames, IEnumerable <IFileInfo> CollectionFileInfos)> CollectDataAsync(CollectorMode collectorMode, DataCollectionConfig dataCollectionConfig, CancellationToken cancellationToken)
{
try
{
(IEnumerable <string> NewFileNames, IEnumerable <IFileInfo> CollectionFileInfos)result;
_logger.LogInformation("Started processing data '{dataCollectionName}'", dataCollectionConfig.DataCollectionName);
var watch = System.Diagnostics.Stopwatch.StartNew();
try
{
result = await _dataCollector.CollectDataAsync(collectorMode, dataCollectionConfig, cancellationToken);
result = (result.NewFileNames.ToList(), result.CollectionFileInfos.ToList());
}
finally
{
watch.Stop();
}
_logger.LogInformation("Finished processing data '{dataCollectionName}'. Elapsed time: {elapsed}", dataCollectionConfig.DataCollectionName, watch.Elapsed);
return(result);
}
catch (Exception e)
{
_logger.LogCritical("Error processing data '{dataCollectionName}': {errorMessage}", dataCollectionConfig.DataCollectionName, e.GetAggregateMessages());
throw;
}
}
public async Task <(IEnumerable <string> NewFileNames, IEnumerable <IFileInfo> CollectionFileInfos)> CollectDataAsync(CollectorMode collectorMode, DataCollectionConfig dataCollectionConfig, CancellationToken cancellationToken)
{
// assert at least one destination before preparing
var destinations = await GetDestinationsAsync(dataCollectionConfig.DestinationIds, cancellationToken);
if (!destinations.Any())
{
throw new Exception("No destinations found");
}
if ((dataCollectionConfig.InitialDelay ?? default).TotalSeconds > 0)
{
await _delay.DelayAsync(dataCollectionConfig.InitialDelay ?? default, $"initial delay for Data '{dataCollectionConfig.DataCollectionName}'", cancellationToken);
}
var collectMoment = DateTimeOffset.Now;
var prepared = (collectorMode != CollectorMode.Collect) ? false : await _dataPreparer.PrepareDataAsync(dataCollectionConfig, cancellationToken);
var collectItems = _collectItemsProvider.GetCollectItems(dataCollectionConfig.DataCollectionName, dataCollectionConfig.CollectFileIdentifiersUrl, dataCollectionConfig.CollectFileIdentifiersHeaders, dataCollectionConfig.CollectUrl, dataCollectionConfig.CollectHeaders, dataCollectionConfig.IdentityServiceClientInfo, cancellationToken).ToList();
var collectionFileInfos = collectItems.Select(x => x.CollectFileInfo);
var acceptedCollectItems = await GetAcceptedCollectItemsAsync(collectItems, dataCollectionConfig.DataCollectionName, destinations, dataCollectionConfig.CollectParallelFileCount ?? 1, cancellationToken);
if (collectorMode == CollectorMode.Collect)
{
if (prepared && (!acceptedCollectItems.Any()))
{
throw new Exception("No data prepared for collecting");
}
if (dataCollectionConfig.MaxFileCount.HasValue)
{
acceptedCollectItems = acceptedCollectItems.Take(dataCollectionConfig.MaxFileCount.Value);
}
var redirectedCollectItems = await _collectItemsProvider.GetRedirectedCollectItemsAsync(acceptedCollectItems, dataCollectionConfig.DataCollectionName, dataCollectionConfig.CollectHeaders, dataCollectionConfig.CollectParallelFileCount ?? 1, dataCollectionConfig.IdentityServiceClientInfo, cancellationToken);
var newFileNames = await _collectItemsCollector.CollectItemsAsync(redirectedCollectItems, dataCollectionConfig.DataCollectionName, destinations, dataCollectionConfig, collectMoment, cancellationToken);
return(newFileNames, collectionFileInfos);
}
if ((collectorMode == CollectorMode.Check) && acceptedCollectItems.Any())
{
throw new Exception("Found missing data");
}
return(Enumerable.Empty <string>(), collectionFileInfos);
}
