Stream IWebContentCache.GetValue(Uri uri)
{
if (config.IsCachingForcedForHost(uri.Host.ToLower()))
{
return(rawContentCache.GetValue(MakeCacheKey(uri)));
}
return(null);
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
if (!TryParseUrl(source.Location, out var request))
{
throw new ArgumentException($"Can not parse URL {source.Location}");
}
using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() =>
{
string zipTmpFileName = callback.TempFilesManager.GenerateNewName();
using (var zipStream = new FileStream(zipTmpFileName, FileMode.CreateNew))
using (var cachedStream = contentCache.GetValue(source.Location))
{
if (cachedStream != null)
{
await cachedStream.CopyToAsync(zipStream);
}
else
{
await DownloadAndMakeZip(request, zipStream, callback);
zipStream.Position = 0;
await contentCache.SetValue(source.Location, zipStream);
}
}
return(zipTmpFileName);
}))
{
if (!sharedDownloadTask.IsValueCreator)
{
callback.SetStepDescription("Waiting for downloaded data...");
}
var tmpFileName = await sharedDownloadTask.Value;
return(new PreprocessingStepParams(
tmpFileName,
source.FullPath,
source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName))
));
}
}
