async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
if (!TryParseUrl(source.Location, out var request))
{
throw new ArgumentException($"Can not parse URL {source.Location}");
}
using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() =>
{
var logs = await CloudWatchDownloader.Download(
webViewTools, request, callback.SetStepDescription);
string zipTmpFileName = callback.TempFilesManager.GenerateNewName();
using (var zipToOpen = new FileStream(zipTmpFileName, FileMode.CreateNew))
using (var archive = new ZipArchive(zipToOpen, ZipArchiveMode.Create))
{
foreach (var l in logs)
{
string tmpFile = callback.TempFilesManager.GenerateNewName();
File.WriteAllText(tmpFile, l.Value);
archive.CreateEntryFromFile(tmpFile, l.Key);
File.Delete(tmpFile);
}
}
return(zipTmpFileName);
}))
{
if (!sharedDownloadTask.IsValueCreator)
{
callback.SetStepDescription("Waiting for downloaded data...");
}
var tmpFileName = await sharedDownloadTask.Value;
return(new PreprocessingStepParams(
tmpFileName,
source.FullPath,
source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName))
));
}
// todo: cache
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
if (!TryParseUrl(source.Location, out var request))
{
throw new ArgumentException($"Can not parse URL {source.Location}");
}
using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() =>
{
string zipTmpFileName = callback.TempFilesManager.GenerateNewName();
using (var zipStream = new FileStream(zipTmpFileName, FileMode.CreateNew))
using (var cachedStream = contentCache.GetValue(source.Location))
{
if (cachedStream != null)
{
await cachedStream.CopyToAsync(zipStream);
}
else
{
await DownloadAndMakeZip(request, zipStream, callback);
zipStream.Position = 0;
await contentCache.SetValue(source.Location, zipStream);
}
}
return(zipTmpFileName);
}))
{
if (!sharedDownloadTask.IsValueCreator)
{
callback.SetStepDescription("Waiting for downloaded data...");
}
var tmpFileName = await sharedDownloadTask.Value;
return(new PreprocessingStepParams(
tmpFileName,
source.FullPath,
source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName))
));
}
}
