async Task DownloadAndMakeZip(
CloudWatchDownloader.DownloadRequest request,
Stream zipStream,
IPreprocessingStepCallback callback
)
{
var logs = await CloudWatchDownloader.Download(
webViewTools, request, callback.SetStepDescription);
using (var archive = new ZipArchive(zipStream, ZipArchiveMode.Create, leaveOpen: true))
{
foreach (var l in logs)
{
string tmpFile = callback.TempFilesManager.GenerateNewName();
File.WriteAllText(tmpFile, l.Value);
archive.CreateEntryFromFile(tmpFile, l.Key);
File.Delete(tmpFile);
}
}
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
if (!TryParseUrl(source.Location, out var request))
{
throw new ArgumentException($"Can not parse URL {source.Location}");
}
using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() =>
{
var logs = await CloudWatchDownloader.Download(
webViewTools, request, callback.SetStepDescription);
string zipTmpFileName = callback.TempFilesManager.GenerateNewName();
using (var zipToOpen = new FileStream(zipTmpFileName, FileMode.CreateNew))
using (var archive = new ZipArchive(zipToOpen, ZipArchiveMode.Create))
{
foreach (var l in logs)
{
string tmpFile = callback.TempFilesManager.GenerateNewName();
File.WriteAllText(tmpFile, l.Value);
archive.CreateEntryFromFile(tmpFile, l.Key);
File.Delete(tmpFile);
}
}
return(zipTmpFileName);
}))
{
if (!sharedDownloadTask.IsValueCreator)
{
callback.SetStepDescription("Waiting for downloaded data...");
}
var tmpFileName = await sharedDownloadTask.Value;
return(new PreprocessingStepParams(
tmpFileName,
source.FullPath,
source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName))
));
}
// todo: cache
}
