private static void HandleFailure(IPreprocessingStepCallback callback, CredentialsImpl credentials, Exception failure)
{
var trace = callback.Trace;
if (failure != null)
{
trace.Error(failure, "Download failed");
var webException = failure as WebException;
if (webException != null)
{
var httpResponse = webException.Response as HttpWebResponse;
if (httpResponse != null && httpResponse.StatusCode == HttpStatusCode.Unauthorized)
{
trace.Warning("User unauthorized");
var lastCred = credentials.LastRequestedCredential;
if (lastCred != null)
{
trace.Info("Invalidating last requested credentials: {0} {1}", lastCred.Item1, lastCred.Item2);
credentials.CredCache.InvalidateCredentialsCache(lastCred.Item1, lastCred.Item2);
}
}
}
throw failure;
}
}
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Uri);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
callback.SetStepDescription(string.Format("{0}: converting to text", sourceFile.FullPath));
var harTask = Task.Run(() =>
{
try
{
return(HarConvert.DeserializeFromFile(sourceFile.Uri));
}
catch (Newtonsoft.Json.JsonReaderException e)
{
throw new Exception(string.Format("HTTP archive is broken"), e);
}
});
if (await Task.WhenAny(callback.Cancellation.ToTask(), harTask) != harTask)
{
return;
}
await(new Writer()).Write(
() => new FileStream(tmpFileName, FileMode.Create),
s => s.Dispose(),
ToText(harTask.Result, callback.Cancellation)
);
onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\text", sourceFile.FullPath),
Utils.Concat(sourceFile.PreprocessingSteps, stepName), sourceFile.FullPath));
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Location);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var sourceFileInfo = new FileInfo(sourceFile.Location);
using (var inFileStream = sourceFileInfo.OpenRead())
using (var outFileStream = new FileStream(tmpFileName, FileMode.CreateNew))
using (var progress = sourceFileInfo.Length != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
{
using (var gzipStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(inFileStream))
{
IOUtils.CopyStreamWithProgress(gzipStream, outFileStream, bytes =>
{
callback.SetStepDescription(string.Format("{1} {0}: Gunzipping...",
IOUtils.FileSizeToString(bytes), sourceFile.FullPath));
if (progress != null)
{
progress.SetValue((double)inFileStream.Position / (double)sourceFileInfo.Length);
}
}, callback.Cancellation);
return
(new PreprocessingStepParams(tmpFileName, sourceFile.FullPath,
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(name))));
}
}
}
async Task ExecuteInternal(IPreprocessingStepCallback callback, Func <PreprocessingStepParams, bool> onNext)
{
await callback.BecomeLongRunning();
string specificFileToExtract = @params.Argument;
callback.TempFilesCleanupList.Add(@params.Location);
for (string password = null;;)
{
try
{
await DoExtract(callback, specificFileToExtract, onNext, password, fileSystem);
break;
}
catch (PasswordException)
{
var uri = new Uri(@params.Location);
var authMethod = "protected-archive";
if (password != null)
{
credCache.InvalidateCredentialsCache(uri, authMethod);
}
var cred = credCache.QueryCredentials(uri, authMethod);
if (cred == null)
{
break;
}
password = cred.Password;
}
}
}
Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback)
{
var header = new StreamHeader(sourceFile.Location);
var detectedFormatStep = extentions.Items.Select(d => d.DetectFormat(sourceFile, header)).FirstOrDefault(x => x != null);
if (detectedFormatStep != null)
{
callback.YieldNextStep(detectedFormatStep);
}
else if (IsZip(sourceFile, header))
{
callback.YieldNextStep(preprocessingStepsFactory.CreateUnpackingStep(sourceFile));
}
else if (IsGzip(sourceFile, header))
{
callback.YieldNextStep(preprocessingStepsFactory.CreateGunzippingStep(sourceFile));
}
else if (IsTar(sourceFile, header))
{
callback.YieldNextStep(preprocessingStepsFactory.CreateUntarStep(sourceFile));
}
else
{
AutodetectFormatAndYield(sourceFile, callback);
}
return(Task.FromResult(0));
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Uri);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var sourceFileInfo = new FileInfo(sourceFile.Uri);
using (var inFileStream = sourceFileInfo.OpenRead())
using (var outFileStream = new FileStream(tmpFileName, FileMode.CreateNew))
using (var progress = sourceFileInfo.Length != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
{
using (var gzipStream = new GZipStream(inFileStream, CompressionMode.Decompress, true))
{
IOUtils.CopyStreamWithProgress(gzipStream, outFileStream, downloadedBytes =>
{
callback.SetStepDescription(string.Format("{1} {0}: Gunzipping...",
IOUtils.FileSizeToString(downloadedBytes), sourceFile.FullPath));
if (progress != null)
{
progress.SetValue((double)downloadedBytes / (double)sourceFileInfo.Length);
}
});
return
(new PreprocessingStepParams(tmpFileName, sourceFile.FullPath,
Utils.Concat(sourceFile.PreprocessingSteps, name)));
}
}
}
async Task ExecuteInternal(IPreprocessingStepCallback callback, string specificFileToExtract, Func <PreprocessingStepParams, bool> onNext)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Uri);
using (var zipFile = new Ionic.Zip.ZipFile(sourceFile.Uri))
{
string currentEntryBeingExtracted = null;
Progress.IProgressEventsSink progress = null;
zipFile.ExtractProgress += (s, evt) =>
{
evt.Cancel = callback.Cancellation.IsCancellationRequested;
if (currentEntryBeingExtracted != null && evt.TotalBytesToTransfer != 0)
{
callback.SetStepDescription(string.Format("Unpacking {1}%: {0}",
currentEntryBeingExtracted,
evt.BytesTransferred * (long)100 / evt.TotalBytesToTransfer));
if (progress != null)
{
progress.SetValue(
(double)evt.BytesTransferred / (double)evt.TotalBytesToTransfer);
}
}
};
var entriesToEnum = specificFileToExtract != null?
Enumerable.Repeat(zipFile[specificFileToExtract], 1) : zipFile.Entries;
foreach (var entry in entriesToEnum.Where(e => e != null))
{
if (entry.IsDirectory)
{
continue;
}
string entryFullPath = sourceFile.FullPath + "\\" + entry.FileName;
string tmpFileName = callback.TempFilesManager.GenerateNewName();
callback.SetStepDescription("Unpacking " + entryFullPath);
using (FileStream tmpFs = new FileStream(tmpFileName, FileMode.CreateNew))
using (var entryProgress = progressAggregator.CreateProgressSink())
{
currentEntryBeingExtracted = entryFullPath;
progress = entryProgress;
entry.Extract(tmpFs);
currentEntryBeingExtracted = null;
progress = null;
}
string preprocessingStep = string.Format("{0} {1}", name, entry.FileName);
if (!onNext(new PreprocessingStepParams(tmpFileName, entryFullPath,
Utils.Concat(sourceFile.PreprocessingSteps, preprocessingStep))))
{
break;
}
}
}
}
async Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback)
{
await ExecuteInternal(callback, null, p =>
{
callback.YieldNextStep(preprocessingStepsFactory.CreateFormatDetectionStep(p));
return(true);
});
}
async Task <PreprocessingStepParams> IPreprocessingStep.ExecuteLoadedStep(IPreprocessingStepCallback callback)
{
PreprocessingStepParams ret = null;
await ExecuteInternal(callback, x => { ret = x; });
return(ret);
}
private void DoExtract(
IPreprocessingStepCallback callback,
string specificFileToExtract,
Func <PreprocessingStepParams, bool> onNext,
string password)
{
using (var zipFile = new ICSharpCode.SharpZipLib.Zip.ZipFile(@params.Location))
{
if (password != null)
{
zipFile.Password = password;
}
var entriesToEnum = specificFileToExtract != null?
Enumerable.Repeat(zipFile.GetEntry(specificFileToExtract), 1) : zipFile.OfType <ICSharpCode.SharpZipLib.Zip.ZipEntry>();
foreach (var entry in entriesToEnum.Where(e => e != null))
{
if (entry.IsDirectory)
{
continue;
}
if (entry.IsCrypted && password == null)
{
throw new PasswordException();
}
string entryFullPath = @params.FullPath + "\\" + entry.Name;
string tmpFileName = callback.TempFilesManager.GenerateNewName();
callback.SetStepDescription("Unpacking " + entryFullPath);
using (FileStream tmpFs = new FileStream(tmpFileName, FileMode.CreateNew))
using (var entryProgress = progressAggregator.CreateProgressSink())
{
using (var entryStream = zipFile.GetInputStream(entry))
{
var totalLen = entry.Size;
IOUtils.CopyStreamWithProgress(entryStream, tmpFs, pos =>
{
if (totalLen > 0)
{
callback.SetStepDescription($"Unpacking {pos * 100 / totalLen}%: {entryFullPath}");
entryProgress.SetValue((double)pos / totalLen);
}
}, callback.Cancellation);
}
}
if (!onNext(new PreprocessingStepParams(tmpFileName, entryFullPath,
@params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, entry.Name)))))
{
break;
}
}
}
}
public void Setup()
{
workspacesManager = Substitute.For <IWorkspacesManager>();
appLaunch = Substitute.For <ILaunchUrlParser>();
preprocessingStepsFactory = Substitute.For <IStepsFactory>();
extensions = Substitute.For <IExtensionsRegistry>();
preprocessingStepsFactory.CreateURLTypeDetectionStep(null).ReturnsForAnyArgs(
callInfo => new URLTypeDetectionStep(
callInfo.Arg <PreprocessingStepParams>(), preprocessingStepsFactory, workspacesManager, appLaunch, extensions));
callback = Substitute.For <IPreprocessingStepCallback>();
}
Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback)
{
if (Uri.IsWellFormedUriString(sourceFile.Uri, UriKind.Absolute))
{
callback.YieldNextStep(preprocessingStepsFactory.CreateURLTypeDetectionStep(sourceFile));
}
else
{
callback.YieldNextStep(preprocessingStepsFactory.CreateFormatDetectionStep(sourceFile));
}
return(Task.FromResult(0));
}
async Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
callback.SetStepDescription("Opening workspace " + source.FullPath);
callback.SetOption(PreprocessingOptions.SkipLogsSelectionDialog, true);
foreach (var entry in await await invoke.Invoke(() => workspacesManager.LoadWorkspace(source.Uri, callback.Cancellation), callback.Cancellation))
{
callback.YieldChildPreprocessing(entry.Log, entry.IsHiddenLog);
}
}
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Location);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
await Converters.JsonToLog(sourceFile.Location, tmpFileName);
onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\converted_to_log", sourceFile.FullPath),
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName))));
}
async Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback)
{
await ExecuteInternal(callback, p =>
{
var cp = ((IFileBasedLogProviderFactory)harLogsFactory).CreateParams(p.Location);
p.DumpToConnectionParams(cp);
callback.YieldLogProvider(new YieldedProvider()
{
Factory = harLogsFactory,
ConnectionParams = cp,
DisplayName = p.DisplayName,
});
});
}
Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback)
{
var prepState = state.preprocessings.GetOrAdd(callback.Owner, _ => new PreprocessingState());
if (pcap != null)
{
prepState.pcaps.Add(pcap);
}
else if (key != null)
{
prepState.keys.Add(key);
}
return(Task.FromResult(0));
}
async Task ExecuteInternal(
IPreprocessingStepCallback callback,
Action <PreprocessingStepParams> yieldOutput)
{
await callback.BecomeLongRunning();
string filter = @params.Argument;
callback.TempFilesCleanupList.Add(@params.Location);
string tmpDirectory = callback.TempFilesManager.GenerateNewName();
var sourceFileInfo = new FileInfo(@params.Location);
using (var inFileStream = sourceFileInfo.OpenRead())
// using (var progress = sourceFileInfo.Length != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
{
using (var tarArchive = TarArchive.CreateInputTarArchive(inFileStream))
{
tarArchive.ExtractContents(tmpDirectory);
}
void traverseFolder(string relativePath)
{
var dirInfo = new DirectoryInfo(Path.Combine(tmpDirectory, relativePath));
foreach (var f in dirInfo.EnumerateFiles())
{
var fileNameInArchive =
relativePath != "" ? $"{relativePath}{Path.DirectorySeparatorChar}{f.Name}" : f.Name;
if (filter == null || filter == fileNameInArchive)
{
yieldOutput(new PreprocessingStepParams(f.FullName,
$"{@params.FullPath}{Path.DirectorySeparatorChar}{fileNameInArchive}",
@params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, fileNameInArchive))));
}
}
foreach (var f in dirInfo.EnumerateDirectories())
{
traverseFolder(Path.Combine(relativePath, f.Name));
}
}
traverseFolder("");
}
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback, PreprocessingStepParams[] keyFiles)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Location);
callback.SetStepDescription("scanning...");
string tmpFileName = callback.TempFilesManager.GenerateNewName();
await Converters.PcapToPdmp(sourceFile.Location, keyFiles.Select(f => f.Location).ToArray(),
tmpFileName, tshark, callback.Cancellation, callback.SetStepDescription, callback.Trace);
return(new PreprocessingStepParams(
tmpFileName,
$"{sourceFile.FullPath}\\as_pdml",
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName, StepArgument.ToString(keyFiles))),
$"{sourceFile.FullPath} (converted to PDML)"
));
}
async Task DownloadAndMakeZip(
CloudWatchDownloader.DownloadRequest request,
Stream zipStream,
IPreprocessingStepCallback callback
)
{
var logs = await CloudWatchDownloader.Download(
webViewTools, request, callback.SetStepDescription);
using (var archive = new ZipArchive(zipStream, ZipArchiveMode.Create, leaveOpen: true))
{
foreach (var l in logs)
{
string tmpFile = callback.TempFilesManager.GenerateNewName();
File.WriteAllText(tmpFile, l.Value);
archive.CreateEntryFromFile(tmpFile, l.Key);
File.Delete(tmpFile);
}
}
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
if (!TryParseUrl(source.Location, out var request))
{
throw new ArgumentException($"Can not parse URL {source.Location}");
}
using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() =>
{
string zipTmpFileName = callback.TempFilesManager.GenerateNewName();
using (var zipStream = new FileStream(zipTmpFileName, FileMode.CreateNew))
using (var cachedStream = contentCache.GetValue(source.Location))
{
if (cachedStream != null)
{
await cachedStream.CopyToAsync(zipStream);
}
else
{
await DownloadAndMakeZip(request, zipStream, callback);
zipStream.Position = 0;
await contentCache.SetValue(source.Location, zipStream);
}
}
return(zipTmpFileName);
}))
{
if (!sharedDownloadTask.IsValueCreator)
{
callback.SetStepDescription("Waiting for downloaded data...");
}
var tmpFileName = await sharedDownloadTask.Value;
return(new PreprocessingStepParams(
tmpFileName,
source.FullPath,
source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName))
));
}
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
if (!TryParseUrl(source.Location, out var request))
{
throw new ArgumentException($"Can not parse URL {source.Location}");
}
using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() =>
{
var logs = await CloudWatchDownloader.Download(
webViewTools, request, callback.SetStepDescription);
string zipTmpFileName = callback.TempFilesManager.GenerateNewName();
using (var zipToOpen = new FileStream(zipTmpFileName, FileMode.CreateNew))
using (var archive = new ZipArchive(zipToOpen, ZipArchiveMode.Create))
{
foreach (var l in logs)
{
string tmpFile = callback.TempFilesManager.GenerateNewName();
File.WriteAllText(tmpFile, l.Value);
archive.CreateEntryFromFile(tmpFile, l.Key);
File.Delete(tmpFile);
}
}
return(zipTmpFileName);
}))
{
if (!sharedDownloadTask.IsValueCreator)
{
callback.SetStepDescription("Waiting for downloaded data...");
}
var tmpFileName = await sharedDownloadTask.Value;
return(new PreprocessingStepParams(
tmpFileName,
source.FullPath,
source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName))
));
}
// todo: cache
}
static void AutodetectFormatAndYield(PreprocessingStepParams file, IPreprocessingStepCallback callback)
{
callback.SetStepDescription(string.Format("Detecting format: {0}", file.FullPath));
var progressHandler = new ProgressHandler()
{
callback = callback
};
var detectedFormat = callback.FormatAutodetect.DetectFormat(file.Location, file.FullPath, progressHandler.cancellation.Token, progressHandler);
if (detectedFormat != null)
{
file.DumpToConnectionParams(detectedFormat.ConnectParams);
callback.YieldLogProvider(new YieldedProvider()
{
Factory = detectedFormat.Factory,
ConnectionParams = detectedFormat.ConnectParams,
DisplayName = file.FullPath,
IsHiddenLog = false
});
}
}
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Uri);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
await(new Writer()).Write(
() => new FileStream(tmpFileName, FileMode.Create),
s => s.Dispose(),
FixTimestamps((new Reader(callback.Cancellation)).Read(
sourceFile.Uri,
progressHandler: prct => callback.SetStepDescription(
string.Format("{0}: fixing timestamps {1}%", sourceFile.FullPath, (int)(prct * 100)))
))
);
onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\with_fixed_timestamps", sourceFile.FullPath),
Utils.Concat(sourceFile.PreprocessingSteps, stepName)));
}
Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback)
{
if (Uri.IsWellFormedUriString(sourceFile.Location, UriKind.Absolute))
{
var uri = new Uri(sourceFile.Location);
string localFilePath;
AppLaunch.LaunchUriData launchUriData;
IPreprocessingStep extensionStep;
if ((localFilePath = TryDetectLocalFileUri(uri)) != null)
{
callback.YieldNextStep(preprocessingStepsFactory.CreateFormatDetectionStep(
new PreprocessingStepParams(localFilePath, localFilePath, sourceFile.PreprocessingHistory)));
}
else if (workspacesManager.IsWorkspaceUri(uri))
{
callback.YieldNextStep(preprocessingStepsFactory.CreateOpenWorkspaceStep(sourceFile));
}
else if (appLaunch.TryParseLaunchUri(uri, out launchUriData))
{
if (launchUriData.SingleLogUri != null)
{
callback.YieldNextStep(preprocessingStepsFactory.CreateURLTypeDetectionStep(new PreprocessingStepParams(launchUriData.SingleLogUri)));
}
else if (launchUriData.WorkspaceUri != null)
{
callback.YieldNextStep(preprocessingStepsFactory.CreateOpenWorkspaceStep(new PreprocessingStepParams(launchUriData.WorkspaceUri)));
}
}
else if (TryExtensions(uri, out extensionStep))
{
callback.YieldNextStep(extensionStep);
}
else
{
callback.YieldNextStep(preprocessingStepsFactory.CreateDownloadingStep(sourceFile));
}
}
return(Task.FromResult(0));
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
string factoryName = @params.Argument;
callback.TempFilesCleanupList.Add(@params.Location);
Action <double?> setStepDescription = prctComplete =>
{
var str = new StringBuilder();
str.Append(@params.FullPath);
str.Append(": fixing timestamp anomalies...");
if (prctComplete != null)
{
str.AppendFormat(" {0}%", (int)(prctComplete.Value * 100));
}
callback.SetStepDescription(str.ToString());
};
setStepDescription(null);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var factoryNameSplit = factoryName.Split('\\');
if (factoryNameSplit.Length != 2)
{
throw new InvalidFormatException();
}
var factory = logProviderFactoryRegistry.Find(factoryNameSplit[0], factoryNameSplit[1]);
if (factory == null)
{
throw new InvalidDataException("factory not found: " + factoryName);
}
var readerFactory = factory as IMediaBasedReaderFactory;
if (readerFactory == null)
{
throw new InvalidDataException("bad factory: " + factoryName);
}
using (ILogMedia fileMedia = await SimpleFileMedia.Create(fileSystem,
SimpleFileMedia.CreateConnectionParamsFromFileName(@params.Location)))
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (var reader = readerFactory.CreateMessagesReader(
new MediaBasedReaderParams(threads, fileMedia)))
{
var readerImpl = reader as MediaBasedPositionedMessagesReader; // todo: do not use real classes; have stream encoding in an interface.
if (readerImpl == null)
{
throw new InvalidDataException("bad reader was made by factory " + factoryName);
}
await reader.UpdateAvailableBounds(false);
var range = new FileRange.Range(reader.BeginPosition, reader.EndPosition);
double rangeLen = range.Length;
using (var progress = progressAggregator.CreateProgressSink())
using (var writer = new StreamWriter(tmpFileName, false, readerImpl.StreamEncoding))
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition,
flags: MessagesParserFlag.DisableDejitter | MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading)), async parser =>
{
var queue = new VCSKicksCollection.PriorityQueue <IMessage>(
new MessagesComparer(ignoreConnectionIds: true));
Action dequeue = () => writer.WriteLine(queue.Dequeue().RawText.ToString());
double lastPrctComplete = 0;
var cancellation = callback.Cancellation;
for (long msgIdx = 0;; ++msgIdx)
{
if (cancellation.IsCancellationRequested)
{
break;
}
var msg = await parser.ReadNext();
if (msg == null)
{
break;
}
if ((msgIdx % progressUpdateThreshold) == 0 && rangeLen > 0)
{
var prctComplete = (double)(msg.Position - range.Begin) / rangeLen;
progress.SetValue(prctComplete);
if (prctComplete - lastPrctComplete > 0.05)
{
setStepDescription(prctComplete);
lastPrctComplete = prctComplete;
}
}
queue.Enqueue(msg);
if (queue.Count > queueSize)
{
dequeue();
}
}
while (queue.Count > 0)
{
dequeue();
}
});
}
return(new PreprocessingStepParams(
tmpFileName,
@params.FullPath + " (reordered)",
@params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, factoryName))
));
}
async Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback)
{
callback.YieldNextStep(preprocessingStepsFactory.CreateFormatDetectionStep(await ExecuteInternal(callback)));
}
Task <PreprocessingStepParams> IPreprocessingStep.ExecuteLoadedStep(IPreprocessingStepCallback callback, string param)
{
return(ExecuteInternal(callback));
}
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Location);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
callback.SetStepDescription(string.Format("{0}: converting to text", sourceFile.FullPath));
var harTask = Task.Run(() =>
{
Assembly dependencyResolveHandler(object s, ResolveEventArgs e)
{
if (new AssemblyName(e.Name).Name == "Newtonsoft.Json") // HarConvert needs Newtonsoft.Json v6, map it to whatever modern version shipped with the plugin
{
return(typeof(Newtonsoft.Json.JsonReaderException).Assembly);
}
return(null);
}
AppDomain.CurrentDomain.AssemblyResolve += dependencyResolveHandler;
try
{
return(HarConvert.DeserializeFromFile(sourceFile.Location));
}
catch (Newtonsoft.Json.JsonReaderException)
{
string fixedJsonFileName = callback.TempFilesManager.GenerateNewName();
try
{
TryFixJson(sourceFile.Location, fixedJsonFileName);
return(HarConvert.DeserializeFromFile(fixedJsonFileName));
}
catch (Newtonsoft.Json.JsonReaderException e)
{
throw new Exception(string.Format("HTTP archive is broken"), e);
}
finally
{
if (File.Exists(fixedJsonFileName))
{
File.Delete(fixedJsonFileName);
}
}
}
finally
{
AppDomain.CurrentDomain.AssemblyResolve -= dependencyResolveHandler;
}
});
if (await Task.WhenAny(ToTask(callback.Cancellation), harTask) != harTask)
{
return;
}
await(new Writer()).Write(
() => new FileStream(tmpFileName, FileMode.Create),
s => s.Dispose(),
ToText(harTask.Result, callback.Cancellation)
);
onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\text", sourceFile.FullPath),
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName)), sourceFile.FullPath));
}
async Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback)
{
var l = await ExecuteInternal(callback);
callback.YieldNextStep(preprocessingStepsFactory.CreateUnpackingStep(l));
}
async Task <PreprocessingStepParams> IPreprocessingStep.ExecuteLoadedStep(IPreprocessingStepCallback callback)
{
return(await ExecuteInternal(callback));
}
