async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback) { await callback.BecomeLongRunning(); callback.TempFilesCleanupList.Add(sourceFile.Uri); string tmpFileName = callback.TempFilesManager.GenerateNewName(); var sourceFileInfo = new FileInfo(sourceFile.Uri); using (var inFileStream = sourceFileInfo.OpenRead()) using (var outFileStream = new FileStream(tmpFileName, FileMode.CreateNew)) using (var progress = sourceFileInfo.Length != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null) { using (var gzipStream = new GZipStream(inFileStream, CompressionMode.Decompress, true)) { IOUtils.CopyStreamWithProgress(gzipStream, outFileStream, downloadedBytes => { callback.SetStepDescription(string.Format("{1} {0}: Gunzipping...", IOUtils.FileSizeToString(downloadedBytes), sourceFile.FullPath)); if (progress != null) { progress.SetValue((double)downloadedBytes / (double)sourceFileInfo.Length); } }); return (new PreprocessingStepParams(tmpFileName, sourceFile.FullPath, Utils.Concat(sourceFile.PreprocessingSteps, name))); } } }
async Task ExecuteInternal(IPreprocessingStepCallback callback, Func <PreprocessingStepParams, bool> onNext) { await callback.BecomeLongRunning(); string specificFileToExtract = @params.Argument; callback.TempFilesCleanupList.Add(@params.Location); for (string password = null;;) { try { await DoExtract(callback, specificFileToExtract, onNext, password, fileSystem); break; } catch (PasswordException) { var uri = new Uri(@params.Location); var authMethod = "protected-archive"; if (password != null) { credCache.InvalidateCredentialsCache(uri, authMethod); } var cred = credCache.QueryCredentials(uri, authMethod); if (cred == null) { break; } password = cred.Password; } } }
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext) { await callback.BecomeLongRunning(); callback.TempFilesCleanupList.Add(sourceFile.Uri); string tmpFileName = callback.TempFilesManager.GenerateNewName(); callback.SetStepDescription(string.Format("{0}: converting to text", sourceFile.FullPath)); var harTask = Task.Run(() => { try { return(HarConvert.DeserializeFromFile(sourceFile.Uri)); } catch (Newtonsoft.Json.JsonReaderException e) { throw new Exception(string.Format("HTTP archive is broken"), e); } }); if (await Task.WhenAny(callback.Cancellation.ToTask(), harTask) != harTask) { return; } await(new Writer()).Write( () => new FileStream(tmpFileName, FileMode.Create), s => s.Dispose(), ToText(harTask.Result, callback.Cancellation) ); onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\text", sourceFile.FullPath), Utils.Concat(sourceFile.PreprocessingSteps, stepName), sourceFile.FullPath)); }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback) { await callback.BecomeLongRunning(); callback.TempFilesCleanupList.Add(sourceFile.Location); string tmpFileName = callback.TempFilesManager.GenerateNewName(); var sourceFileInfo = new FileInfo(sourceFile.Location); using (var inFileStream = sourceFileInfo.OpenRead()) using (var outFileStream = new FileStream(tmpFileName, FileMode.CreateNew)) using (var progress = sourceFileInfo.Length != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null) { using (var gzipStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(inFileStream)) { IOUtils.CopyStreamWithProgress(gzipStream, outFileStream, bytes => { callback.SetStepDescription(string.Format("{1} {0}: Gunzipping...", IOUtils.FileSizeToString(bytes), sourceFile.FullPath)); if (progress != null) { progress.SetValue((double)inFileStream.Position / (double)sourceFileInfo.Length); } }, callback.Cancellation); return (new PreprocessingStepParams(tmpFileName, sourceFile.FullPath, sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(name)))); } } }
async Task ExecuteInternal(IPreprocessingStepCallback callback, string specificFileToExtract, Func <PreprocessingStepParams, bool> onNext) { await callback.BecomeLongRunning(); callback.TempFilesCleanupList.Add(sourceFile.Uri); using (var zipFile = new Ionic.Zip.ZipFile(sourceFile.Uri)) { string currentEntryBeingExtracted = null; Progress.IProgressEventsSink progress = null; zipFile.ExtractProgress += (s, evt) => { evt.Cancel = callback.Cancellation.IsCancellationRequested; if (currentEntryBeingExtracted != null && evt.TotalBytesToTransfer != 0) { callback.SetStepDescription(string.Format("Unpacking {1}%: {0}", currentEntryBeingExtracted, evt.BytesTransferred * (long)100 / evt.TotalBytesToTransfer)); if (progress != null) { progress.SetValue( (double)evt.BytesTransferred / (double)evt.TotalBytesToTransfer); } } }; var entriesToEnum = specificFileToExtract != null? Enumerable.Repeat(zipFile[specificFileToExtract], 1) : zipFile.Entries; foreach (var entry in entriesToEnum.Where(e => e != null)) { if (entry.IsDirectory) { continue; } string entryFullPath = sourceFile.FullPath + "\\" + entry.FileName; string tmpFileName = callback.TempFilesManager.GenerateNewName(); callback.SetStepDescription("Unpacking " + entryFullPath); using (FileStream tmpFs = new FileStream(tmpFileName, FileMode.CreateNew)) using (var entryProgress = progressAggregator.CreateProgressSink()) { currentEntryBeingExtracted = entryFullPath; progress = entryProgress; entry.Extract(tmpFs); currentEntryBeingExtracted = null; progress = null; } string preprocessingStep = string.Format("{0} {1}", name, entry.FileName); if (!onNext(new PreprocessingStepParams(tmpFileName, entryFullPath, Utils.Concat(sourceFile.PreprocessingSteps, preprocessingStep)))) { break; } } } }
async Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback) { await callback.BecomeLongRunning(); callback.SetStepDescription("Opening workspace " + source.FullPath); callback.SetOption(PreprocessingOptions.SkipLogsSelectionDialog, true); foreach (var entry in await await invoke.Invoke(() => workspacesManager.LoadWorkspace(source.Uri, callback.Cancellation), callback.Cancellation)) { callback.YieldChildPreprocessing(entry.Log, entry.IsHiddenLog); } }
void IFormatAutodetectionProgress.Trying(ILogProviderFactory factory) { formatsTriedSoFar++; if (formatsTriedSoFar == 2) { callback.BecomeLongRunning(); } if (callback.Cancellation.IsCancellationRequested) { cancellation.Cancel(); } }
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext) { await callback.BecomeLongRunning(); callback.TempFilesCleanupList.Add(sourceFile.Location); string tmpFileName = callback.TempFilesManager.GenerateNewName(); await Converters.JsonToLog(sourceFile.Location, tmpFileName); onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\converted_to_log", sourceFile.FullPath), sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName)))); }
async Task ExecuteInternal( IPreprocessingStepCallback callback, Action <PreprocessingStepParams> yieldOutput) { await callback.BecomeLongRunning(); string filter = @params.Argument; callback.TempFilesCleanupList.Add(@params.Location); string tmpDirectory = callback.TempFilesManager.GenerateNewName(); var sourceFileInfo = new FileInfo(@params.Location); using (var inFileStream = sourceFileInfo.OpenRead()) // using (var progress = sourceFileInfo.Length != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null) { using (var tarArchive = TarArchive.CreateInputTarArchive(inFileStream)) { tarArchive.ExtractContents(tmpDirectory); } void traverseFolder(string relativePath) { var dirInfo = new DirectoryInfo(Path.Combine(tmpDirectory, relativePath)); foreach (var f in dirInfo.EnumerateFiles()) { var fileNameInArchive = relativePath != "" ? $"{relativePath}{Path.DirectorySeparatorChar}{f.Name}" : f.Name; if (filter == null || filter == fileNameInArchive) { yieldOutput(new PreprocessingStepParams(f.FullName, $"{@params.FullPath}{Path.DirectorySeparatorChar}{fileNameInArchive}", @params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, fileNameInArchive)))); } } foreach (var f in dirInfo.EnumerateDirectories()) { traverseFolder(Path.Combine(relativePath, f.Name)); } } traverseFolder(""); } }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback, PreprocessingStepParams[] keyFiles) { await callback.BecomeLongRunning(); callback.TempFilesCleanupList.Add(sourceFile.Location); callback.SetStepDescription("scanning..."); string tmpFileName = callback.TempFilesManager.GenerateNewName(); await Converters.PcapToPdmp(sourceFile.Location, keyFiles.Select(f => f.Location).ToArray(), tmpFileName, tshark, callback.Cancellation, callback.SetStepDescription, callback.Trace); return(new PreprocessingStepParams( tmpFileName, $"{sourceFile.FullPath}\\as_pdml", sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName, StepArgument.ToString(keyFiles))), $"{sourceFile.FullPath} (converted to PDML)" )); }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback) { await callback.BecomeLongRunning(); if (!TryParseUrl(source.Location, out var request)) { throw new ArgumentException($"Can not parse URL {source.Location}"); } using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() => { var logs = await CloudWatchDownloader.Download( webViewTools, request, callback.SetStepDescription); string zipTmpFileName = callback.TempFilesManager.GenerateNewName(); using (var zipToOpen = new FileStream(zipTmpFileName, FileMode.CreateNew)) using (var archive = new ZipArchive(zipToOpen, ZipArchiveMode.Create)) { foreach (var l in logs) { string tmpFile = callback.TempFilesManager.GenerateNewName(); File.WriteAllText(tmpFile, l.Value); archive.CreateEntryFromFile(tmpFile, l.Key); File.Delete(tmpFile); } } return(zipTmpFileName); })) { if (!sharedDownloadTask.IsValueCreator) { callback.SetStepDescription("Waiting for downloaded data..."); } var tmpFileName = await sharedDownloadTask.Value; return(new PreprocessingStepParams( tmpFileName, source.FullPath, source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName)) )); } // todo: cache }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback) { await callback.BecomeLongRunning(); if (!TryParseUrl(source.Location, out var request)) { throw new ArgumentException($"Can not parse URL {source.Location}"); } using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() => { string zipTmpFileName = callback.TempFilesManager.GenerateNewName(); using (var zipStream = new FileStream(zipTmpFileName, FileMode.CreateNew)) using (var cachedStream = contentCache.GetValue(source.Location)) { if (cachedStream != null) { await cachedStream.CopyToAsync(zipStream); } else { await DownloadAndMakeZip(request, zipStream, callback); zipStream.Position = 0; await contentCache.SetValue(source.Location, zipStream); } } return(zipTmpFileName); })) { if (!sharedDownloadTask.IsValueCreator) { callback.SetStepDescription("Waiting for downloaded data..."); } var tmpFileName = await sharedDownloadTask.Value; return(new PreprocessingStepParams( tmpFileName, source.FullPath, source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName)) )); } }
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext) { await callback.BecomeLongRunning(); callback.TempFilesCleanupList.Add(sourceFile.Uri); string tmpFileName = callback.TempFilesManager.GenerateNewName(); await(new Writer()).Write( () => new FileStream(tmpFileName, FileMode.Create), s => s.Dispose(), FixTimestamps((new Reader(callback.Cancellation)).Read( sourceFile.Uri, progressHandler: prct => callback.SetStepDescription( string.Format("{0}: fixing timestamps {1}%", sourceFile.FullPath, (int)(prct * 100))) )) ); onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\with_fixed_timestamps", sourceFile.FullPath), Utils.Concat(sourceFile.PreprocessingSteps, stepName))); }
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext) { await callback.BecomeLongRunning(); callback.TempFilesCleanupList.Add(sourceFile.Location); string tmpFileName = callback.TempFilesManager.GenerateNewName(); callback.SetStepDescription(string.Format("{0}: converting to text", sourceFile.FullPath)); var harTask = Task.Run(() => { Assembly dependencyResolveHandler(object s, ResolveEventArgs e) { if (new AssemblyName(e.Name).Name == "Newtonsoft.Json") // HarConvert needs Newtonsoft.Json v6, map it to whatever modern version shipped with the plugin { return(typeof(Newtonsoft.Json.JsonReaderException).Assembly); } return(null); } AppDomain.CurrentDomain.AssemblyResolve += dependencyResolveHandler; try { return(HarConvert.DeserializeFromFile(sourceFile.Location)); } catch (Newtonsoft.Json.JsonReaderException) { string fixedJsonFileName = callback.TempFilesManager.GenerateNewName(); try { TryFixJson(sourceFile.Location, fixedJsonFileName); return(HarConvert.DeserializeFromFile(fixedJsonFileName)); } catch (Newtonsoft.Json.JsonReaderException e) { throw new Exception(string.Format("HTTP archive is broken"), e); } finally { if (File.Exists(fixedJsonFileName)) { File.Delete(fixedJsonFileName); } } } finally { AppDomain.CurrentDomain.AssemblyResolve -= dependencyResolveHandler; } }); if (await Task.WhenAny(ToTask(callback.Cancellation), harTask) != harTask) { return; } await(new Writer()).Write( () => new FileStream(tmpFileName, FileMode.Create), s => s.Dispose(), ToText(harTask.Result, callback.Cancellation) ); onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\text", sourceFile.FullPath), sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName)), sourceFile.FullPath)); }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback) { await callback.BecomeLongRunning(); string factoryName = @params.Argument; callback.TempFilesCleanupList.Add(@params.Location); Action <double?> setStepDescription = prctComplete => { var str = new StringBuilder(); str.Append(@params.FullPath); str.Append(": fixing timestamp anomalies..."); if (prctComplete != null) { str.AppendFormat(" {0}%", (int)(prctComplete.Value * 100)); } callback.SetStepDescription(str.ToString()); }; setStepDescription(null); string tmpFileName = callback.TempFilesManager.GenerateNewName(); var factoryNameSplit = factoryName.Split('\\'); if (factoryNameSplit.Length != 2) { throw new InvalidFormatException(); } var factory = logProviderFactoryRegistry.Find(factoryNameSplit[0], factoryNameSplit[1]); if (factory == null) { throw new InvalidDataException("factory not found: " + factoryName); } var readerFactory = factory as IMediaBasedReaderFactory; if (readerFactory == null) { throw new InvalidDataException("bad factory: " + factoryName); } using (ILogMedia fileMedia = await SimpleFileMedia.Create(fileSystem, SimpleFileMedia.CreateConnectionParamsFromFileName(@params.Location))) using (ILogSourceThreadsInternal threads = new LogSourceThreads()) using (var reader = readerFactory.CreateMessagesReader( new MediaBasedReaderParams(threads, fileMedia))) { var readerImpl = reader as MediaBasedPositionedMessagesReader; // todo: do not use real classes; have stream encoding in an interface. if (readerImpl == null) { throw new InvalidDataException("bad reader was made by factory " + factoryName); } await reader.UpdateAvailableBounds(false); var range = new FileRange.Range(reader.BeginPosition, reader.EndPosition); double rangeLen = range.Length; using (var progress = progressAggregator.CreateProgressSink()) using (var writer = new StreamWriter(tmpFileName, false, readerImpl.StreamEncoding)) await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition, flags: MessagesParserFlag.DisableDejitter | MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading)), async parser => { var queue = new VCSKicksCollection.PriorityQueue <IMessage>( new MessagesComparer(ignoreConnectionIds: true)); Action dequeue = () => writer.WriteLine(queue.Dequeue().RawText.ToString()); double lastPrctComplete = 0; var cancellation = callback.Cancellation; for (long msgIdx = 0;; ++msgIdx) { if (cancellation.IsCancellationRequested) { break; } var msg = await parser.ReadNext(); if (msg == null) { break; } if ((msgIdx % progressUpdateThreshold) == 0 && rangeLen > 0) { var prctComplete = (double)(msg.Position - range.Begin) / rangeLen; progress.SetValue(prctComplete); if (prctComplete - lastPrctComplete > 0.05) { setStepDescription(prctComplete); lastPrctComplete = prctComplete; } } queue.Enqueue(msg); if (queue.Count > queueSize) { dequeue(); } } while (queue.Count > 0) { dequeue(); } }); } return(new PreprocessingStepParams( tmpFileName, @params.FullPath + " (reordered)", @params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, factoryName)) )); }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback) { var trace = callback.Trace; using (trace.NewFrame) { await callback.BecomeLongRunning(); trace.Info("Downloading '{0}' from '{1}'", sourceFile.FullPath, sourceFile.Location); callback.SetStepDescription("Downloading " + sourceFile.FullPath); string tmpFileName = callback.TempFilesManager.GenerateNewName(); trace.Info("Temporary filename to download to: {0}", tmpFileName); Func <Stream, long, string, Task> writeToTempFile = async(fromStream, contentLength, description) => { using (FileStream fs = new FileStream(tmpFileName, FileMode.Create)) using (var progress = contentLength != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null) { await IOUtils.CopyStreamWithProgressAsync(fromStream, fs, downloadedBytes => { callback.SetStepDescription(string.Format("{2} {0}: {1}", IOUtils.FileSizeToString(downloadedBytes), sourceFile.FullPath, description)); if (progress != null) { progress.SetValue((double)downloadedBytes / (double)contentLength); } }, callback.Cancellation); } }; var uri = new Uri(sourceFile.Location); LogDownloaderRule logDownloaderRule; using (var cachedValue = cache.GetValue(uri)) { if (cachedValue != null) { await writeToTempFile(cachedValue, cachedValue.Length, "Loading from cache"); } else if ((logDownloaderRule = config.GetLogDownloaderConfig(uri)) != null && logDownloaderRule.UseWebBrowserDownloader) { using (var stream = await webBrowserDownloader.Download(new WebViewTools.DownloadParams() { Location = uri, ExpectedMimeType = logDownloaderRule.ExpectedMimeType, Cancellation = callback.Cancellation, Progress = progressAggregator, IsLoginUrl = testUri => logDownloaderRule.LoginUrls.Any(loginUrl => testUri.GetLeftPart(UriPartial.Path).Contains(loginUrl)) })) { await writeToTempFile(stream, 0, "Downloading"); } } else if (useHttpClient) { using (var client = new System.Net.Http.HttpClient()) { trace.Info("Start downloading {0}", sourceFile.Location); var response = await client.GetAsync(uri); await writeToTempFile(await response.Content.ReadAsStreamAsync(), response.Content.Headers.ContentLength.GetValueOrDefault(0), "Downloading"); } } else { using (WebClient client = new WebClient()) using (ManualResetEvent completed = new ManualResetEvent(false)) { ServicePointManager.ServerCertificateValidationCallback = delegate { return(true); }; var credentials = new CredentialsImpl() { Callback = callback, CredCache = credCache }; client.Credentials = credentials; Exception failure = null; client.OpenReadCompleted += (s, evt) => { failure = evt.Error; if (failure != null) { trace.Error(failure, "Downloading {0} completed with error", sourceFile.Location); } if (failure == null && (evt.Cancelled || callback.Cancellation.IsCancellationRequested)) { trace.Warning("Downloading {0} cancelled", sourceFile.Location); failure = new Exception("Aborted"); } if (failure == null) { try { long contentLength; long.TryParse(client.ResponseHeaders["Content-Length"] ?? "", out contentLength); writeToTempFile(evt.Result, contentLength, "Downloading").Wait(); } catch (Exception e) { trace.Error(e, "Failed saving to file"); failure = e; } } completed.Set(); }; trace.Info("Start downloading {0}", sourceFile.Location); client.OpenReadAsync(uri); if (WaitHandle.WaitAny(new WaitHandle[] { completed, callback.Cancellation.WaitHandle }) == 1) { trace.Info("Cancellation event was triggered. Cancelling download."); client.CancelAsync(); completed.WaitOne(); } HandleFailure(callback, credentials, failure); using (FileStream fs = new FileStream(tmpFileName, FileMode.Open)) { cache.SetValue(new Uri(sourceFile.Location), fs).Wait(); } } } } string preprocessingStep = name; return(new PreprocessingStepParams( tmpFileName, sourceFile.FullPath, sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(preprocessingStep)) )); } }