async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Location);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var sourceFileInfo = new FileInfo(sourceFile.Location);
using (var inFileStream = sourceFileInfo.OpenRead())
using (var outFileStream = new FileStream(tmpFileName, FileMode.CreateNew))
using (var progress = sourceFileInfo.Length != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
{
using (var gzipStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(inFileStream))
{
IOUtils.CopyStreamWithProgress(gzipStream, outFileStream, bytes =>
{
callback.SetStepDescription(string.Format("{1} {0}: Gunzipping...",
IOUtils.FileSizeToString(bytes), sourceFile.FullPath));
if (progress != null)
{
progress.SetValue((double)inFileStream.Position / (double)sourceFileInfo.Length);
}
}, callback.Cancellation);
return
(new PreprocessingStepParams(tmpFileName, sourceFile.FullPath,
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(name))));
}
}
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Uri);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var sourceFileInfo = new FileInfo(sourceFile.Uri);
using (var inFileStream = sourceFileInfo.OpenRead())
using (var outFileStream = new FileStream(tmpFileName, FileMode.CreateNew))
using (var progress = sourceFileInfo.Length != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
{
using (var gzipStream = new GZipStream(inFileStream, CompressionMode.Decompress, true))
{
IOUtils.CopyStreamWithProgress(gzipStream, outFileStream, downloadedBytes =>
{
callback.SetStepDescription(string.Format("{1} {0}: Gunzipping...",
IOUtils.FileSizeToString(downloadedBytes), sourceFile.FullPath));
if (progress != null)
{
progress.SetValue((double)downloadedBytes / (double)sourceFileInfo.Length);
}
});
return
(new PreprocessingStepParams(tmpFileName, sourceFile.FullPath,
Utils.Concat(sourceFile.PreprocessingSteps, name)));
}
}
}
public async Task CanReportIssue()
{
traceAccess.When(x => x.ClearMemBufferAndGetCurrentContents(Arg.Any <TextWriter>())).Do(call =>
{
var w = call.Arg <TextWriter>();
w.Write("line 1\n");
w.Write("line 2\n");
});
using (var uploadedStreamContents = new MemoryStream())
{
uploader.UploadIssueReport(Arg.Do <Stream>(s =>
{
IOUtils.CopyStreamWithProgress(s, uploadedStreamContents, _ => { }, CancellationToken.None);
}), Arg.Any <CancellationToken>()).ReturnsForAnyArgs(Task.FromResult("https://blobs/123"));
await collector.ReportIssue("something went wrong for me");
uploadedStreamContents.Position = 0;
using (var zip = new ZipFile(uploadedStreamContents))
{
Assert.AreEqual(30, zip.GetEntry("description.txt").Size);
Assert.AreEqual(17, zip.GetEntry("membuffer.log").Size);
}
}
}
private void DoExtract(
IPreprocessingStepCallback callback,
string specificFileToExtract,
Func <PreprocessingStepParams, bool> onNext,
string password)
{
using (var zipFile = new ICSharpCode.SharpZipLib.Zip.ZipFile(@params.Location))
{
if (password != null)
{
zipFile.Password = password;
}
var entriesToEnum = specificFileToExtract != null?
Enumerable.Repeat(zipFile.GetEntry(specificFileToExtract), 1) : zipFile.OfType <ICSharpCode.SharpZipLib.Zip.ZipEntry>();
foreach (var entry in entriesToEnum.Where(e => e != null))
{
if (entry.IsDirectory)
{
continue;
}
if (entry.IsCrypted && password == null)
{
throw new PasswordException();
}
string entryFullPath = @params.FullPath + "\\" + entry.Name;
string tmpFileName = callback.TempFilesManager.GenerateNewName();
callback.SetStepDescription("Unpacking " + entryFullPath);
using (FileStream tmpFs = new FileStream(tmpFileName, FileMode.CreateNew))
using (var entryProgress = progressAggregator.CreateProgressSink())
{
using (var entryStream = zipFile.GetInputStream(entry))
{
var totalLen = entry.Size;
IOUtils.CopyStreamWithProgress(entryStream, tmpFs, pos =>
{
if (totalLen > 0)
{
callback.SetStepDescription($"Unpacking {pos * 100 / totalLen}%: {entryFullPath}");
entryProgress.SetValue((double)pos / totalLen);
}
}, callback.Cancellation);
}
}
if (!onNext(new PreprocessingStepParams(tmpFileName, entryFullPath,
@params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, entry.Name)))))
{
break;
}
}
}
}
private async Task <string> CreateEntriesArchive(List <KeyValuePair <string, Stream> > entriesToArchive)
{
var entriesArchiveFileName = tempFilesManager.GenerateNewName();
await Task.Run(() =>
{
using (var baseOutputStream = new FileStream(entriesArchiveFileName, FileMode.Create))
using (var zip = new ZipOutputStream(baseOutputStream))
{
foreach (var entry in entriesToArchive)
{
var newEntry = new ZipEntry(entry.Key);
zip.PutNextEntry(newEntry);
entry.Value.Position = 0;
IOUtils.CopyStreamWithProgress(entry.Value, zip, _ => { }, CancellationToken.None);
zip.CloseEntry();
}
}
});
return(entriesArchiveFileName);
}
private async Task LoadArchivedStorageEntries(string entriesArchiveUrl, CancellationToken cancellation)
{
var entriesArchiveFileName = tempFilesManager.GenerateNewName();
using (var entriesArchiveStream = new FileStream(entriesArchiveFileName, FileMode.CreateNew))
{
await backendAccess.GetEntriesArchive(entriesArchiveUrl, entriesArchiveStream, cancellation);
}
var sectionContentTempFileName = tempFilesManager.GenerateNewName();
var entries = new Dictionary <string, IStorageEntry>();
using (var zipFile = new ZipFile(entriesArchiveFileName))
{
foreach (var zipEntry in zipFile.OfType <ZipEntry>().Where(e => e != null))
{
if (zipEntry.IsDirectory)
{
continue;
}
var storageEntryId = Path.GetDirectoryName(zipEntry.Name);
var sectionId = Path.GetFileName(zipEntry.Name);
using (var sectionContentStream = new FileStream(sectionContentTempFileName,
FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose))
{
using (var inputStream = zipFile.GetInputStream(zipEntry))
IOUtils.CopyStreamWithProgress(inputStream, sectionContentStream, _ => { }, CancellationToken.None);
sectionContentStream.Position = 0;
IStorageEntry storageEntry;
if (!entries.TryGetValue(storageEntryId, out storageEntry))
{
entries.Add(storageEntryId, storageEntry = storageManager.GetEntryById(storageEntryId));
}
await storageEntry.LoadSectionFromSnapshot(sectionId, sectionContentStream, cancellation);
}
}
}
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
var trace = callback.Trace;
using (trace.NewFrame)
{
await callback.BecomeLongRunning();
trace.Info("Downloading '{0}' from '{1}'", sourceFile.FullPath, sourceFile.Location);
callback.SetStepDescription("Downloading " + sourceFile.FullPath);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
trace.Info("Temporary filename to download to: {0}", tmpFileName);
Action <Stream, long, string> writeToTempFile = (fromStream, contentLength, description) =>
{
using (FileStream fs = new FileStream(tmpFileName, FileMode.Create))
using (var progress = contentLength != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
{
IOUtils.CopyStreamWithProgress(fromStream, fs, downloadedBytes =>
{
callback.SetStepDescription(string.Format("{2} {0}: {1}",
IOUtils.FileSizeToString(downloadedBytes), sourceFile.FullPath, description));
if (progress != null)
{
progress.SetValue((double)downloadedBytes / (double)contentLength);
}
}, callback.Cancellation);
}
};
var uri = new Uri(sourceFile.Location);
LogDownloaderRule logDownloaderRule;
using (var cachedValue = cache.GetValue(uri))
{
if (cachedValue != null)
{
writeToTempFile(cachedValue, cachedValue.Length, "Loading from cache");
}
else if ((logDownloaderRule = config.GetLogDownloaderConfig(uri)) != null && logDownloaderRule.UseWebBrowserDownloader)
{
using (var stream = await webBrowserDownloader.Download(new WebViewTools.DownloadParams()
{
Location = uri,
ExpectedMimeType = logDownloaderRule.ExpectedMimeType,
Cancellation = callback.Cancellation,
Progress = progressAggregator,
IsLoginUrl = testUri => logDownloaderRule.LoginUrls.Any(loginUrl => testUri.GetLeftPart(UriPartial.Path).Contains(loginUrl))
}))
{
writeToTempFile(stream, 0, "Downloading");
}
}
else
{
using (WebClient client = new WebClient())
using (ManualResetEvent completed = new ManualResetEvent(false))
{
ServicePointManager.ServerCertificateValidationCallback = delegate { return(true); };
var credentials = new CredentialsImpl()
{
Callback = callback, CredCache = credCache
};
client.Credentials = credentials;
Exception failure = null;
client.OpenReadCompleted += (s, evt) =>
{
failure = evt.Error;
if (failure != null)
{
trace.Error(failure, "Downloading {0} completed with error", sourceFile.Location);
}
if (failure == null && (evt.Cancelled || callback.Cancellation.IsCancellationRequested))
{
trace.Warning("Downloading {0} cancelled", sourceFile.Location);
failure = new Exception("Aborted");
}
if (failure == null)
{
try
{
long contentLength;
long.TryParse(client.ResponseHeaders["Content-Length"] ?? "", out contentLength);
writeToTempFile(evt.Result, contentLength, "Downloading");
}
catch (Exception e)
{
trace.Error(e, "Failed saving to file");
failure = e;
}
}
completed.Set();
};
trace.Info("Start downloading {0}", sourceFile.Location);
client.OpenReadAsync(uri);
if (WaitHandle.WaitAny(new WaitHandle[] { completed, callback.Cancellation.WaitHandle }) == 1)
{
trace.Info("Cancellation event was triggered. Cancelling download.");
client.CancelAsync();
completed.WaitOne();
}
HandleFailure(callback, credentials, failure);
using (FileStream fs = new FileStream(tmpFileName, FileMode.Open))
{
cache.SetValue(new Uri(sourceFile.Location), fs).Wait();
}
}
}
}
string preprocessingStep = name;
return(new PreprocessingStepParams(
tmpFileName, sourceFile.FullPath,
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(preprocessingStep))
));
}
}
