private async Task DoExtract(
IPreprocessingStepCallback callback,
string specificFileToExtract,
Func <PreprocessingStepParams, bool> onNext,
string password,
IFileSystem fileSystem)
{
async Task <ICSharpCode.SharpZipLib.Zip.ZipFile> CreateZipFile()
{
if (!IsBrowser.Value)
{
return(new ICSharpCode.SharpZipLib.Zip.ZipFile(@params.Location));
}
else
{
var ms = new MemoryStream();
using (var fileStream = await fileSystem.OpenFile(@params.Location))
await IOUtils.CopyStreamWithProgressAsync(fileStream, ms, _ => { }, callback.Cancellation);
return(new ICSharpCode.SharpZipLib.Zip.ZipFile(ms, leaveOpen: false));
}
}
using (var zipFile = await CreateZipFile())
{
if (password != null)
{
zipFile.Password = password;
}
var entriesToEnum = specificFileToExtract != null?
Enumerable.Repeat(zipFile.GetEntry(specificFileToExtract), 1) : zipFile.OfType <ICSharpCode.SharpZipLib.Zip.ZipEntry>();
foreach (var entry in entriesToEnum.Where(e => e != null))
{
if (entry.IsDirectory)
{
continue;
}
if (entry.IsCrypted && password == null)
{
throw new PasswordException();
}
string entryFullPath = @params.FullPath + "\\" + entry.Name;
string tmpFileName = callback.TempFilesManager.GenerateNewName();
callback.SetStepDescription("Unpacking " + entryFullPath);
using (FileStream tmpFs = new FileStream(tmpFileName, FileMode.CreateNew))
using (var entryProgress = progressAggregator.CreateProgressSink())
{
using (var entryStream = zipFile.GetInputStream(entry))
{
var totalLen = entry.Size;
IOUtils.CopyStreamWithProgress(entryStream, tmpFs, pos =>
{
if (totalLen > 0)
{
callback.SetStepDescription($"Unpacking {pos * 100 / totalLen}%: {entryFullPath}");
entryProgress.SetValue((double)pos / totalLen);
}
}, callback.Cancellation);
}
}
if (!onNext(new PreprocessingStepParams(tmpFileName, entryFullPath,
@params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, entry.Name)))))
{
break;
}
}
}
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
var trace = callback.Trace;
using (trace.NewFrame)
{
await callback.BecomeLongRunning();
trace.Info("Downloading '{0}' from '{1}'", sourceFile.FullPath, sourceFile.Location);
callback.SetStepDescription("Downloading " + sourceFile.FullPath);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
trace.Info("Temporary filename to download to: {0}", tmpFileName);
Func <Stream, long, string, Task> writeToTempFile = async(fromStream, contentLength, description) =>
{
using (FileStream fs = new FileStream(tmpFileName, FileMode.Create))
using (var progress = contentLength != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
{
await IOUtils.CopyStreamWithProgressAsync(fromStream, fs, downloadedBytes =>
{
callback.SetStepDescription(string.Format("{2} {0}: {1}",
IOUtils.FileSizeToString(downloadedBytes), sourceFile.FullPath, description));
if (progress != null)
{
progress.SetValue((double)downloadedBytes / (double)contentLength);
}
}, callback.Cancellation);
}
};
var uri = new Uri(sourceFile.Location);
LogDownloaderRule logDownloaderRule;
using (var cachedValue = cache.GetValue(uri))
{
if (cachedValue != null)
{
await writeToTempFile(cachedValue, cachedValue.Length, "Loading from cache");
}
else if ((logDownloaderRule = config.GetLogDownloaderConfig(uri)) != null && logDownloaderRule.UseWebBrowserDownloader)
{
using (var stream = await webBrowserDownloader.Download(new WebViewTools.DownloadParams()
{
Location = uri,
ExpectedMimeType = logDownloaderRule.ExpectedMimeType,
Cancellation = callback.Cancellation,
Progress = progressAggregator,
IsLoginUrl = testUri => logDownloaderRule.LoginUrls.Any(loginUrl => testUri.GetLeftPart(UriPartial.Path).Contains(loginUrl))
}))
{
await writeToTempFile(stream, 0, "Downloading");
}
}
else if (useHttpClient)
{
using (var client = new System.Net.Http.HttpClient())
{
trace.Info("Start downloading {0}", sourceFile.Location);
var response = await client.GetAsync(uri);
await writeToTempFile(await response.Content.ReadAsStreamAsync(),
response.Content.Headers.ContentLength.GetValueOrDefault(0), "Downloading");
}
}
else
{
using (WebClient client = new WebClient())
using (ManualResetEvent completed = new ManualResetEvent(false))
{
ServicePointManager.ServerCertificateValidationCallback = delegate { return(true); };
var credentials = new CredentialsImpl()
{
Callback = callback, CredCache = credCache
};
client.Credentials = credentials;
Exception failure = null;
client.OpenReadCompleted += (s, evt) =>
{
failure = evt.Error;
if (failure != null)
{
trace.Error(failure, "Downloading {0} completed with error", sourceFile.Location);
}
if (failure == null && (evt.Cancelled || callback.Cancellation.IsCancellationRequested))
{
trace.Warning("Downloading {0} cancelled", sourceFile.Location);
failure = new Exception("Aborted");
}
if (failure == null)
{
try
{
long contentLength;
long.TryParse(client.ResponseHeaders["Content-Length"] ?? "", out contentLength);
writeToTempFile(evt.Result, contentLength, "Downloading").Wait();
}
catch (Exception e)
{
trace.Error(e, "Failed saving to file");
failure = e;
}
}
completed.Set();
};
trace.Info("Start downloading {0}", sourceFile.Location);
client.OpenReadAsync(uri);
if (WaitHandle.WaitAny(new WaitHandle[] { completed, callback.Cancellation.WaitHandle }) == 1)
{
trace.Info("Cancellation event was triggered. Cancelling download.");
client.CancelAsync();
completed.WaitOne();
}
HandleFailure(callback, credentials, failure);
using (FileStream fs = new FileStream(tmpFileName, FileMode.Open))
{
cache.SetValue(new Uri(sourceFile.Location), fs).Wait();
}
}
}
}
string preprocessingStep = name;
return(new PreprocessingStepParams(
tmpFileName, sourceFile.FullPath,
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(preprocessingStep))
));
}
}
