async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Uri);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var sourceFileInfo = new FileInfo(sourceFile.Uri);
using (var inFileStream = sourceFileInfo.OpenRead())
using (var outFileStream = new FileStream(tmpFileName, FileMode.CreateNew))
using (var progress = sourceFileInfo.Length != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
{
using (var gzipStream = new GZipStream(inFileStream, CompressionMode.Decompress, true))
{
IOUtils.CopyStreamWithProgress(gzipStream, outFileStream, downloadedBytes =>
{
callback.SetStepDescription(string.Format("{1} {0}: Gunzipping...",
IOUtils.FileSizeToString(downloadedBytes), sourceFile.FullPath));
if (progress != null)
{
progress.SetValue((double)downloadedBytes / (double)sourceFileInfo.Length);
}
});
return
(new PreprocessingStepParams(tmpFileName, sourceFile.FullPath,
Utils.Concat(sourceFile.PreprocessingSteps, name)));
}
}
}
async Task ExecuteInternal(IPreprocessingStepCallback callback, Func <PreprocessingStepParams, bool> onNext)
{
await callback.BecomeLongRunning();
string specificFileToExtract = @params.Argument;
callback.TempFilesCleanupList.Add(@params.Location);
for (string password = null;;)
{
try
{
await DoExtract(callback, specificFileToExtract, onNext, password, fileSystem);
break;
}
catch (PasswordException)
{
var uri = new Uri(@params.Location);
var authMethod = "protected-archive";
if (password != null)
{
credCache.InvalidateCredentialsCache(uri, authMethod);
}
var cred = credCache.QueryCredentials(uri, authMethod);
if (cred == null)
{
break;
}
password = cred.Password;
}
}
}
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Uri);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
callback.SetStepDescription(string.Format("{0}: converting to text", sourceFile.FullPath));
var harTask = Task.Run(() =>
{
try
{
return(HarConvert.DeserializeFromFile(sourceFile.Uri));
}
catch (Newtonsoft.Json.JsonReaderException e)
{
throw new Exception(string.Format("HTTP archive is broken"), e);
}
});
if (await Task.WhenAny(callback.Cancellation.ToTask(), harTask) != harTask)
{
return;
}
await(new Writer()).Write(
() => new FileStream(tmpFileName, FileMode.Create),
s => s.Dispose(),
ToText(harTask.Result, callback.Cancellation)
);
onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\text", sourceFile.FullPath),
Utils.Concat(sourceFile.PreprocessingSteps, stepName), sourceFile.FullPath));
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Location);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var sourceFileInfo = new FileInfo(sourceFile.Location);
using (var inFileStream = sourceFileInfo.OpenRead())
using (var outFileStream = new FileStream(tmpFileName, FileMode.CreateNew))
using (var progress = sourceFileInfo.Length != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
{
using (var gzipStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(inFileStream))
{
IOUtils.CopyStreamWithProgress(gzipStream, outFileStream, bytes =>
{
callback.SetStepDescription(string.Format("{1} {0}: Gunzipping...",
IOUtils.FileSizeToString(bytes), sourceFile.FullPath));
if (progress != null)
{
progress.SetValue((double)inFileStream.Position / (double)sourceFileInfo.Length);
}
}, callback.Cancellation);
return
(new PreprocessingStepParams(tmpFileName, sourceFile.FullPath,
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(name))));
}
}
}
async Task ExecuteInternal(IPreprocessingStepCallback callback, string specificFileToExtract, Func <PreprocessingStepParams, bool> onNext)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Uri);
using (var zipFile = new Ionic.Zip.ZipFile(sourceFile.Uri))
{
string currentEntryBeingExtracted = null;
Progress.IProgressEventsSink progress = null;
zipFile.ExtractProgress += (s, evt) =>
{
evt.Cancel = callback.Cancellation.IsCancellationRequested;
if (currentEntryBeingExtracted != null && evt.TotalBytesToTransfer != 0)
{
callback.SetStepDescription(string.Format("Unpacking {1}%: {0}",
currentEntryBeingExtracted,
evt.BytesTransferred * (long)100 / evt.TotalBytesToTransfer));
if (progress != null)
{
progress.SetValue(
(double)evt.BytesTransferred / (double)evt.TotalBytesToTransfer);
}
}
};
var entriesToEnum = specificFileToExtract != null?
Enumerable.Repeat(zipFile[specificFileToExtract], 1) : zipFile.Entries;
foreach (var entry in entriesToEnum.Where(e => e != null))
{
if (entry.IsDirectory)
{
continue;
}
string entryFullPath = sourceFile.FullPath + "\\" + entry.FileName;
string tmpFileName = callback.TempFilesManager.GenerateNewName();
callback.SetStepDescription("Unpacking " + entryFullPath);
using (FileStream tmpFs = new FileStream(tmpFileName, FileMode.CreateNew))
using (var entryProgress = progressAggregator.CreateProgressSink())
{
currentEntryBeingExtracted = entryFullPath;
progress = entryProgress;
entry.Extract(tmpFs);
currentEntryBeingExtracted = null;
progress = null;
}
string preprocessingStep = string.Format("{0} {1}", name, entry.FileName);
if (!onNext(new PreprocessingStepParams(tmpFileName, entryFullPath,
Utils.Concat(sourceFile.PreprocessingSteps, preprocessingStep))))
{
break;
}
}
}
}
async Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
callback.SetStepDescription("Opening workspace " + source.FullPath);
callback.SetOption(PreprocessingOptions.SkipLogsSelectionDialog, true);
foreach (var entry in await await invoke.Invoke(() => workspacesManager.LoadWorkspace(source.Uri, callback.Cancellation), callback.Cancellation))
{
callback.YieldChildPreprocessing(entry.Log, entry.IsHiddenLog);
}
}
void IFormatAutodetectionProgress.Trying(ILogProviderFactory factory)
{
formatsTriedSoFar++;
if (formatsTriedSoFar == 2)
{
callback.BecomeLongRunning();
}
if (callback.Cancellation.IsCancellationRequested)
{
cancellation.Cancel();
}
}
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Location);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
await Converters.JsonToLog(sourceFile.Location, tmpFileName);
onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\converted_to_log", sourceFile.FullPath),
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName))));
}
async Task ExecuteInternal(
IPreprocessingStepCallback callback,
Action <PreprocessingStepParams> yieldOutput)
{
await callback.BecomeLongRunning();
string filter = @params.Argument;
callback.TempFilesCleanupList.Add(@params.Location);
string tmpDirectory = callback.TempFilesManager.GenerateNewName();
var sourceFileInfo = new FileInfo(@params.Location);
using (var inFileStream = sourceFileInfo.OpenRead())
// using (var progress = sourceFileInfo.Length != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
{
using (var tarArchive = TarArchive.CreateInputTarArchive(inFileStream))
{
tarArchive.ExtractContents(tmpDirectory);
}
void traverseFolder(string relativePath)
{
var dirInfo = new DirectoryInfo(Path.Combine(tmpDirectory, relativePath));
foreach (var f in dirInfo.EnumerateFiles())
{
var fileNameInArchive =
relativePath != "" ? $"{relativePath}{Path.DirectorySeparatorChar}{f.Name}" : f.Name;
if (filter == null || filter == fileNameInArchive)
{
yieldOutput(new PreprocessingStepParams(f.FullName,
$"{@params.FullPath}{Path.DirectorySeparatorChar}{fileNameInArchive}",
@params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, fileNameInArchive))));
}
}
foreach (var f in dirInfo.EnumerateDirectories())
{
traverseFolder(Path.Combine(relativePath, f.Name));
}
}
traverseFolder("");
}
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback, PreprocessingStepParams[] keyFiles)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Location);
callback.SetStepDescription("scanning...");
string tmpFileName = callback.TempFilesManager.GenerateNewName();
await Converters.PcapToPdmp(sourceFile.Location, keyFiles.Select(f => f.Location).ToArray(),
tmpFileName, tshark, callback.Cancellation, callback.SetStepDescription, callback.Trace);
return(new PreprocessingStepParams(
tmpFileName,
$"{sourceFile.FullPath}\\as_pdml",
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName, StepArgument.ToString(keyFiles))),
$"{sourceFile.FullPath} (converted to PDML)"
));
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
if (!TryParseUrl(source.Location, out var request))
{
throw new ArgumentException($"Can not parse URL {source.Location}");
}
using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() =>
{
var logs = await CloudWatchDownloader.Download(
webViewTools, request, callback.SetStepDescription);
string zipTmpFileName = callback.TempFilesManager.GenerateNewName();
using (var zipToOpen = new FileStream(zipTmpFileName, FileMode.CreateNew))
using (var archive = new ZipArchive(zipToOpen, ZipArchiveMode.Create))
{
foreach (var l in logs)
{
string tmpFile = callback.TempFilesManager.GenerateNewName();
File.WriteAllText(tmpFile, l.Value);
archive.CreateEntryFromFile(tmpFile, l.Key);
File.Delete(tmpFile);
}
}
return(zipTmpFileName);
}))
{
if (!sharedDownloadTask.IsValueCreator)
{
callback.SetStepDescription("Waiting for downloaded data...");
}
var tmpFileName = await sharedDownloadTask.Value;
return(new PreprocessingStepParams(
tmpFileName,
source.FullPath,
source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName))
));
}
// todo: cache
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
if (!TryParseUrl(source.Location, out var request))
{
throw new ArgumentException($"Can not parse URL {source.Location}");
}
using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() =>
{
string zipTmpFileName = callback.TempFilesManager.GenerateNewName();
using (var zipStream = new FileStream(zipTmpFileName, FileMode.CreateNew))
using (var cachedStream = contentCache.GetValue(source.Location))
{
if (cachedStream != null)
{
await cachedStream.CopyToAsync(zipStream);
}
else
{
await DownloadAndMakeZip(request, zipStream, callback);
zipStream.Position = 0;
await contentCache.SetValue(source.Location, zipStream);
}
}
return(zipTmpFileName);
}))
{
if (!sharedDownloadTask.IsValueCreator)
{
callback.SetStepDescription("Waiting for downloaded data...");
}
var tmpFileName = await sharedDownloadTask.Value;
return(new PreprocessingStepParams(
tmpFileName,
source.FullPath,
source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName))
));
}
}
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Uri);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
await(new Writer()).Write(
() => new FileStream(tmpFileName, FileMode.Create),
s => s.Dispose(),
FixTimestamps((new Reader(callback.Cancellation)).Read(
sourceFile.Uri,
progressHandler: prct => callback.SetStepDescription(
string.Format("{0}: fixing timestamps {1}%", sourceFile.FullPath, (int)(prct * 100)))
))
);
onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\with_fixed_timestamps", sourceFile.FullPath),
Utils.Concat(sourceFile.PreprocessingSteps, stepName)));
}
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Location);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
callback.SetStepDescription(string.Format("{0}: converting to text", sourceFile.FullPath));
var harTask = Task.Run(() =>
{
Assembly dependencyResolveHandler(object s, ResolveEventArgs e)
{
if (new AssemblyName(e.Name).Name == "Newtonsoft.Json") // HarConvert needs Newtonsoft.Json v6, map it to whatever modern version shipped with the plugin
{
return(typeof(Newtonsoft.Json.JsonReaderException).Assembly);
}
return(null);
}
AppDomain.CurrentDomain.AssemblyResolve += dependencyResolveHandler;
try
{
return(HarConvert.DeserializeFromFile(sourceFile.Location));
}
catch (Newtonsoft.Json.JsonReaderException)
{
string fixedJsonFileName = callback.TempFilesManager.GenerateNewName();
try
{
TryFixJson(sourceFile.Location, fixedJsonFileName);
return(HarConvert.DeserializeFromFile(fixedJsonFileName));
}
catch (Newtonsoft.Json.JsonReaderException e)
{
throw new Exception(string.Format("HTTP archive is broken"), e);
}
finally
{
if (File.Exists(fixedJsonFileName))
{
File.Delete(fixedJsonFileName);
}
}
}
finally
{
AppDomain.CurrentDomain.AssemblyResolve -= dependencyResolveHandler;
}
});
if (await Task.WhenAny(ToTask(callback.Cancellation), harTask) != harTask)
{
return;
}
await(new Writer()).Write(
() => new FileStream(tmpFileName, FileMode.Create),
s => s.Dispose(),
ToText(harTask.Result, callback.Cancellation)
);
onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\text", sourceFile.FullPath),
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName)), sourceFile.FullPath));
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
string factoryName = @params.Argument;
callback.TempFilesCleanupList.Add(@params.Location);
Action <double?> setStepDescription = prctComplete =>
{
var str = new StringBuilder();
str.Append(@params.FullPath);
str.Append(": fixing timestamp anomalies...");
if (prctComplete != null)
{
str.AppendFormat(" {0}%", (int)(prctComplete.Value * 100));
}
callback.SetStepDescription(str.ToString());
};
setStepDescription(null);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var factoryNameSplit = factoryName.Split('\\');
if (factoryNameSplit.Length != 2)
{
throw new InvalidFormatException();
}
var factory = logProviderFactoryRegistry.Find(factoryNameSplit[0], factoryNameSplit[1]);
if (factory == null)
{
throw new InvalidDataException("factory not found: " + factoryName);
}
var readerFactory = factory as IMediaBasedReaderFactory;
if (readerFactory == null)
{
throw new InvalidDataException("bad factory: " + factoryName);
}
using (ILogMedia fileMedia = await SimpleFileMedia.Create(fileSystem,
SimpleFileMedia.CreateConnectionParamsFromFileName(@params.Location)))
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (var reader = readerFactory.CreateMessagesReader(
new MediaBasedReaderParams(threads, fileMedia)))
{
var readerImpl = reader as MediaBasedPositionedMessagesReader; // todo: do not use real classes; have stream encoding in an interface.
if (readerImpl == null)
{
throw new InvalidDataException("bad reader was made by factory " + factoryName);
}
await reader.UpdateAvailableBounds(false);
var range = new FileRange.Range(reader.BeginPosition, reader.EndPosition);
double rangeLen = range.Length;
using (var progress = progressAggregator.CreateProgressSink())
using (var writer = new StreamWriter(tmpFileName, false, readerImpl.StreamEncoding))
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition,
flags: MessagesParserFlag.DisableDejitter | MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading)), async parser =>
{
var queue = new VCSKicksCollection.PriorityQueue <IMessage>(
new MessagesComparer(ignoreConnectionIds: true));
Action dequeue = () => writer.WriteLine(queue.Dequeue().RawText.ToString());
double lastPrctComplete = 0;
var cancellation = callback.Cancellation;
for (long msgIdx = 0;; ++msgIdx)
{
if (cancellation.IsCancellationRequested)
{
break;
}
var msg = await parser.ReadNext();
if (msg == null)
{
break;
}
if ((msgIdx % progressUpdateThreshold) == 0 && rangeLen > 0)
{
var prctComplete = (double)(msg.Position - range.Begin) / rangeLen;
progress.SetValue(prctComplete);
if (prctComplete - lastPrctComplete > 0.05)
{
setStepDescription(prctComplete);
lastPrctComplete = prctComplete;
}
}
queue.Enqueue(msg);
if (queue.Count > queueSize)
{
dequeue();
}
}
while (queue.Count > 0)
{
dequeue();
}
});
}
return(new PreprocessingStepParams(
tmpFileName,
@params.FullPath + " (reordered)",
@params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, factoryName))
));
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
var trace = callback.Trace;
using (trace.NewFrame)
{
await callback.BecomeLongRunning();
trace.Info("Downloading '{0}' from '{1}'", sourceFile.FullPath, sourceFile.Location);
callback.SetStepDescription("Downloading " + sourceFile.FullPath);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
trace.Info("Temporary filename to download to: {0}", tmpFileName);
Func <Stream, long, string, Task> writeToTempFile = async(fromStream, contentLength, description) =>
{
using (FileStream fs = new FileStream(tmpFileName, FileMode.Create))
using (var progress = contentLength != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
{
await IOUtils.CopyStreamWithProgressAsync(fromStream, fs, downloadedBytes =>
{
callback.SetStepDescription(string.Format("{2} {0}: {1}",
IOUtils.FileSizeToString(downloadedBytes), sourceFile.FullPath, description));
if (progress != null)
{
progress.SetValue((double)downloadedBytes / (double)contentLength);
}
}, callback.Cancellation);
}
};
var uri = new Uri(sourceFile.Location);
LogDownloaderRule logDownloaderRule;
using (var cachedValue = cache.GetValue(uri))
{
if (cachedValue != null)
{
await writeToTempFile(cachedValue, cachedValue.Length, "Loading from cache");
}
else if ((logDownloaderRule = config.GetLogDownloaderConfig(uri)) != null && logDownloaderRule.UseWebBrowserDownloader)
{
using (var stream = await webBrowserDownloader.Download(new WebViewTools.DownloadParams()
{
Location = uri,
ExpectedMimeType = logDownloaderRule.ExpectedMimeType,
Cancellation = callback.Cancellation,
Progress = progressAggregator,
IsLoginUrl = testUri => logDownloaderRule.LoginUrls.Any(loginUrl => testUri.GetLeftPart(UriPartial.Path).Contains(loginUrl))
}))
{
await writeToTempFile(stream, 0, "Downloading");
}
}
else if (useHttpClient)
{
using (var client = new System.Net.Http.HttpClient())
{
trace.Info("Start downloading {0}", sourceFile.Location);
var response = await client.GetAsync(uri);
await writeToTempFile(await response.Content.ReadAsStreamAsync(),
response.Content.Headers.ContentLength.GetValueOrDefault(0), "Downloading");
}
}
else
{
using (WebClient client = new WebClient())
using (ManualResetEvent completed = new ManualResetEvent(false))
{
ServicePointManager.ServerCertificateValidationCallback = delegate { return(true); };
var credentials = new CredentialsImpl()
{
Callback = callback, CredCache = credCache
};
client.Credentials = credentials;
Exception failure = null;
client.OpenReadCompleted += (s, evt) =>
{
failure = evt.Error;
if (failure != null)
{
trace.Error(failure, "Downloading {0} completed with error", sourceFile.Location);
}
if (failure == null && (evt.Cancelled || callback.Cancellation.IsCancellationRequested))
{
trace.Warning("Downloading {0} cancelled", sourceFile.Location);
failure = new Exception("Aborted");
}
if (failure == null)
{
try
{
long contentLength;
long.TryParse(client.ResponseHeaders["Content-Length"] ?? "", out contentLength);
writeToTempFile(evt.Result, contentLength, "Downloading").Wait();
}
catch (Exception e)
{
trace.Error(e, "Failed saving to file");
failure = e;
}
}
completed.Set();
};
trace.Info("Start downloading {0}", sourceFile.Location);
client.OpenReadAsync(uri);
if (WaitHandle.WaitAny(new WaitHandle[] { completed, callback.Cancellation.WaitHandle }) == 1)
{
trace.Info("Cancellation event was triggered. Cancelling download.");
client.CancelAsync();
completed.WaitOne();
}
HandleFailure(callback, credentials, failure);
using (FileStream fs = new FileStream(tmpFileName, FileMode.Open))
{
cache.SetValue(new Uri(sourceFile.Location), fs).Wait();
}
}
}
}
string preprocessingStep = name;
return(new PreprocessingStepParams(
tmpFileName, sourceFile.FullPath,
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(preprocessingStep))
));
}
}