public IEnumerable <(IFileInfo CollectFileInfo, string CollectUrl)> GetCollectItems(string dataCollectionName, string collectFileIdentifiersUrl, IEnumerable <KeyValuePair <string, string> > collectFileIdentifiersHeaders, string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo identityServiceClientInfo, CancellationToken cancellationToken) { try { _logger.LogInformation("Started getting collect items for data collection '{dataCollectionName}'", dataCollectionName); var result = _collectItemsProvider.GetCollectItems(dataCollectionName, collectFileIdentifiersUrl, collectFileIdentifiersHeaders, collectUrl, collectHeaders, identityServiceClientInfo, cancellationToken) .OrderBy(x => x.CollectFileInfo?.Name) .ToList(); _logger.LogInformation("Finished getting collect items for data collection '{dataCollectionName}'", dataCollectionName); _logger.LogInformation($"Retrieved {result.Count()} collect items for data collection '{dataCollectionName}'", dataCollectionName); foreach (var collectItem in result) { _logger.LogInformation(collectItem.CollectFileInfo?.Name); } return(result); } catch (Exception e) { _logger.LogCritical("Error getting collect items for data collection '{dataCollectionName}': {errorMessage}", dataCollectionName, e.GetAggregateMessages()); throw; } }
public async Task <(IEnumerable <string> NewFileNames, IEnumerable <IFileInfo> CollectionFileInfos)> CollectDataAsync(CollectorMode collectorMode, DataCollectionConfig dataCollectionConfig, CancellationToken cancellationToken) { // assert at least one destination before preparing var destinations = await GetDestinationsAsync(dataCollectionConfig.DestinationIds, cancellationToken); if (!destinations.Any()) { throw new Exception("No destinations found"); } if ((dataCollectionConfig.InitialDelay ?? default).TotalSeconds > 0) { await _delay.DelayAsync(dataCollectionConfig.InitialDelay ?? default, $"initial delay for Data '{dataCollectionConfig.DataCollectionName}'", cancellationToken); } var collectMoment = DateTimeOffset.Now; var prepared = (collectorMode != CollectorMode.Collect) ? false : await _dataPreparer.PrepareDataAsync(dataCollectionConfig, cancellationToken); var collectItems = _collectItemsProvider.GetCollectItems(dataCollectionConfig.DataCollectionName, dataCollectionConfig.CollectFileIdentifiersUrl, dataCollectionConfig.CollectFileIdentifiersHeaders, dataCollectionConfig.CollectUrl, dataCollectionConfig.CollectHeaders, dataCollectionConfig.IdentityServiceClientInfo, cancellationToken).ToList(); var collectionFileInfos = collectItems.Select(x => x.CollectFileInfo); var acceptedCollectItems = await GetAcceptedCollectItemsAsync(collectItems, dataCollectionConfig.DataCollectionName, destinations, dataCollectionConfig.CollectParallelFileCount ?? 1, cancellationToken); if (collectorMode == CollectorMode.Collect) { if (prepared && (!acceptedCollectItems.Any())) { throw new Exception("No data prepared for collecting"); } if (dataCollectionConfig.MaxFileCount.HasValue) { acceptedCollectItems = acceptedCollectItems.Take(dataCollectionConfig.MaxFileCount.Value); } var redirectedCollectItems = await _collectItemsProvider.GetRedirectedCollectItemsAsync(acceptedCollectItems, dataCollectionConfig.DataCollectionName, dataCollectionConfig.CollectHeaders, dataCollectionConfig.CollectParallelFileCount ?? 1, dataCollectionConfig.IdentityServiceClientInfo, cancellationToken); var newFileNames = await _collectItemsCollector.CollectItemsAsync(redirectedCollectItems, dataCollectionConfig.DataCollectionName, destinations, dataCollectionConfig, collectMoment, cancellationToken); return(newFileNames, collectionFileInfos); } if ((collectorMode == CollectorMode.Check) && acceptedCollectItems.Any()) { throw new Exception("Found missing data"); } return(Enumerable.Empty <string>(), collectionFileInfos); }