async Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback) { await callback.BecomeLongRunning(); callback.SetStepDescription("Opening workspace " + source.FullPath); callback.SetOption(PreprocessingOptions.SkipLogsSelectionDialog, true); foreach (var entry in await await invoke.Invoke(() => workspacesManager.LoadWorkspace(source.Uri, callback.Cancellation), callback.Cancellation)) { callback.YieldChildPreprocessing(entry.Log, entry.IsHiddenLog); } }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback, PreprocessingStepParams[] keyFiles) { await callback.BecomeLongRunning(); callback.TempFilesCleanupList.Add(sourceFile.Location); callback.SetStepDescription("scanning..."); string tmpFileName = callback.TempFilesManager.GenerateNewName(); await Converters.PcapToPdmp(sourceFile.Location, keyFiles.Select(f => f.Location).ToArray(), tmpFileName, tshark, callback.Cancellation, callback.SetStepDescription, callback.Trace); return(new PreprocessingStepParams( tmpFileName, $"{sourceFile.FullPath}\\as_pdml", sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName, StepArgument.ToString(keyFiles))), $"{sourceFile.FullPath} (converted to PDML)" )); }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback) { await callback.BecomeLongRunning(); if (!TryParseUrl(source.Location, out var request)) { throw new ArgumentException($"Can not parse URL {source.Location}"); } using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() => { var logs = await CloudWatchDownloader.Download( webViewTools, request, callback.SetStepDescription); string zipTmpFileName = callback.TempFilesManager.GenerateNewName(); using (var zipToOpen = new FileStream(zipTmpFileName, FileMode.CreateNew)) using (var archive = new ZipArchive(zipToOpen, ZipArchiveMode.Create)) { foreach (var l in logs) { string tmpFile = callback.TempFilesManager.GenerateNewName(); File.WriteAllText(tmpFile, l.Value); archive.CreateEntryFromFile(tmpFile, l.Key); File.Delete(tmpFile); } } return(zipTmpFileName); })) { if (!sharedDownloadTask.IsValueCreator) { callback.SetStepDescription("Waiting for downloaded data..."); } var tmpFileName = await sharedDownloadTask.Value; return(new PreprocessingStepParams( tmpFileName, source.FullPath, source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName)) )); } // todo: cache }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback) { await callback.BecomeLongRunning(); if (!TryParseUrl(source.Location, out var request)) { throw new ArgumentException($"Can not parse URL {source.Location}"); } using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() => { string zipTmpFileName = callback.TempFilesManager.GenerateNewName(); using (var zipStream = new FileStream(zipTmpFileName, FileMode.CreateNew)) using (var cachedStream = contentCache.GetValue(source.Location)) { if (cachedStream != null) { await cachedStream.CopyToAsync(zipStream); } else { await DownloadAndMakeZip(request, zipStream, callback); zipStream.Position = 0; await contentCache.SetValue(source.Location, zipStream); } } return(zipTmpFileName); })) { if (!sharedDownloadTask.IsValueCreator) { callback.SetStepDescription("Waiting for downloaded data..."); } var tmpFileName = await sharedDownloadTask.Value; return(new PreprocessingStepParams( tmpFileName, source.FullPath, source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName)) )); } }
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext) { await callback.BecomeLongRunning(); callback.TempFilesCleanupList.Add(sourceFile.Uri); string tmpFileName = callback.TempFilesManager.GenerateNewName(); await(new Writer()).Write( () => new FileStream(tmpFileName, FileMode.Create), s => s.Dispose(), FixTimestamps((new Reader(callback.Cancellation)).Read( sourceFile.Uri, progressHandler: prct => callback.SetStepDescription( string.Format("{0}: fixing timestamps {1}%", sourceFile.FullPath, (int)(prct * 100))) )) ); onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\with_fixed_timestamps", sourceFile.FullPath), Utils.Concat(sourceFile.PreprocessingSteps, stepName))); }
static void AutodetectFormatAndYield(PreprocessingStepParams file, IPreprocessingStepCallback callback) { callback.SetStepDescription(string.Format("Detecting format: {0}", file.FullPath)); var progressHandler = new ProgressHandler() { callback = callback }; var detectedFormat = callback.FormatAutodetect.DetectFormat(file.Location, file.FullPath, progressHandler.cancellation.Token, progressHandler); if (detectedFormat != null) { file.DumpToConnectionParams(detectedFormat.ConnectParams); callback.YieldLogProvider(new YieldedProvider() { Factory = detectedFormat.Factory, ConnectionParams = detectedFormat.ConnectParams, DisplayName = file.FullPath, IsHiddenLog = false }); } }
async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext) { await callback.BecomeLongRunning(); callback.TempFilesCleanupList.Add(sourceFile.Location); string tmpFileName = callback.TempFilesManager.GenerateNewName(); callback.SetStepDescription(string.Format("{0}: converting to text", sourceFile.FullPath)); var harTask = Task.Run(() => { Assembly dependencyResolveHandler(object s, ResolveEventArgs e) { if (new AssemblyName(e.Name).Name == "Newtonsoft.Json") // HarConvert needs Newtonsoft.Json v6, map it to whatever modern version shipped with the plugin { return(typeof(Newtonsoft.Json.JsonReaderException).Assembly); } return(null); } AppDomain.CurrentDomain.AssemblyResolve += dependencyResolveHandler; try { return(HarConvert.DeserializeFromFile(sourceFile.Location)); } catch (Newtonsoft.Json.JsonReaderException) { string fixedJsonFileName = callback.TempFilesManager.GenerateNewName(); try { TryFixJson(sourceFile.Location, fixedJsonFileName); return(HarConvert.DeserializeFromFile(fixedJsonFileName)); } catch (Newtonsoft.Json.JsonReaderException e) { throw new Exception(string.Format("HTTP archive is broken"), e); } finally { if (File.Exists(fixedJsonFileName)) { File.Delete(fixedJsonFileName); } } } finally { AppDomain.CurrentDomain.AssemblyResolve -= dependencyResolveHandler; } }); if (await Task.WhenAny(ToTask(callback.Cancellation), harTask) != harTask) { return; } await(new Writer()).Write( () => new FileStream(tmpFileName, FileMode.Create), s => s.Dispose(), ToText(harTask.Result, callback.Cancellation) ); onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\text", sourceFile.FullPath), sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName)), sourceFile.FullPath)); }
private async Task DoExtract( IPreprocessingStepCallback callback, string specificFileToExtract, Func <PreprocessingStepParams, bool> onNext, string password, IFileSystem fileSystem) { async Task <ICSharpCode.SharpZipLib.Zip.ZipFile> CreateZipFile() { if (!IsBrowser.Value) { return(new ICSharpCode.SharpZipLib.Zip.ZipFile(@params.Location)); } else { var ms = new MemoryStream(); using (var fileStream = await fileSystem.OpenFile(@params.Location)) await IOUtils.CopyStreamWithProgressAsync(fileStream, ms, _ => { }, callback.Cancellation); return(new ICSharpCode.SharpZipLib.Zip.ZipFile(ms, leaveOpen: false)); } } using (var zipFile = await CreateZipFile()) { if (password != null) { zipFile.Password = password; } var entriesToEnum = specificFileToExtract != null? Enumerable.Repeat(zipFile.GetEntry(specificFileToExtract), 1) : zipFile.OfType <ICSharpCode.SharpZipLib.Zip.ZipEntry>(); foreach (var entry in entriesToEnum.Where(e => e != null)) { if (entry.IsDirectory) { continue; } if (entry.IsCrypted && password == null) { throw new PasswordException(); } string entryFullPath = @params.FullPath + "\\" + entry.Name; string tmpFileName = callback.TempFilesManager.GenerateNewName(); callback.SetStepDescription("Unpacking " + entryFullPath); using (FileStream tmpFs = new FileStream(tmpFileName, FileMode.CreateNew)) using (var entryProgress = progressAggregator.CreateProgressSink()) { using (var entryStream = zipFile.GetInputStream(entry)) { var totalLen = entry.Size; IOUtils.CopyStreamWithProgress(entryStream, tmpFs, pos => { if (totalLen > 0) { callback.SetStepDescription($"Unpacking {pos * 100 / totalLen}%: {entryFullPath}"); entryProgress.SetValue((double)pos / totalLen); } }, callback.Cancellation); } } if (!onNext(new PreprocessingStepParams(tmpFileName, entryFullPath, @params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, entry.Name))))) { break; } } } }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback) { await callback.BecomeLongRunning(); string factoryName = @params.Argument; callback.TempFilesCleanupList.Add(@params.Location); Action <double?> setStepDescription = prctComplete => { var str = new StringBuilder(); str.Append(@params.FullPath); str.Append(": fixing timestamp anomalies..."); if (prctComplete != null) { str.AppendFormat(" {0}%", (int)(prctComplete.Value * 100)); } callback.SetStepDescription(str.ToString()); }; setStepDescription(null); string tmpFileName = callback.TempFilesManager.GenerateNewName(); var factoryNameSplit = factoryName.Split('\\'); if (factoryNameSplit.Length != 2) { throw new InvalidFormatException(); } var factory = logProviderFactoryRegistry.Find(factoryNameSplit[0], factoryNameSplit[1]); if (factory == null) { throw new InvalidDataException("factory not found: " + factoryName); } var readerFactory = factory as IMediaBasedReaderFactory; if (readerFactory == null) { throw new InvalidDataException("bad factory: " + factoryName); } using (ILogMedia fileMedia = await SimpleFileMedia.Create(fileSystem, SimpleFileMedia.CreateConnectionParamsFromFileName(@params.Location))) using (ILogSourceThreadsInternal threads = new LogSourceThreads()) using (var reader = readerFactory.CreateMessagesReader( new MediaBasedReaderParams(threads, fileMedia))) { var readerImpl = reader as MediaBasedPositionedMessagesReader; // todo: do not use real classes; have stream encoding in an interface. if (readerImpl == null) { throw new InvalidDataException("bad reader was made by factory " + factoryName); } await reader.UpdateAvailableBounds(false); var range = new FileRange.Range(reader.BeginPosition, reader.EndPosition); double rangeLen = range.Length; using (var progress = progressAggregator.CreateProgressSink()) using (var writer = new StreamWriter(tmpFileName, false, readerImpl.StreamEncoding)) await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition, flags: MessagesParserFlag.DisableDejitter | MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading)), async parser => { var queue = new VCSKicksCollection.PriorityQueue <IMessage>( new MessagesComparer(ignoreConnectionIds: true)); Action dequeue = () => writer.WriteLine(queue.Dequeue().RawText.ToString()); double lastPrctComplete = 0; var cancellation = callback.Cancellation; for (long msgIdx = 0;; ++msgIdx) { if (cancellation.IsCancellationRequested) { break; } var msg = await parser.ReadNext(); if (msg == null) { break; } if ((msgIdx % progressUpdateThreshold) == 0 && rangeLen > 0) { var prctComplete = (double)(msg.Position - range.Begin) / rangeLen; progress.SetValue(prctComplete); if (prctComplete - lastPrctComplete > 0.05) { setStepDescription(prctComplete); lastPrctComplete = prctComplete; } } queue.Enqueue(msg); if (queue.Count > queueSize) { dequeue(); } } while (queue.Count > 0) { dequeue(); } }); } return(new PreprocessingStepParams( tmpFileName, @params.FullPath + " (reordered)", @params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, factoryName)) )); }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback) { var trace = callback.Trace; using (trace.NewFrame) { await callback.BecomeLongRunning(); trace.Info("Downloading '{0}' from '{1}'", sourceFile.FullPath, sourceFile.Location); callback.SetStepDescription("Downloading " + sourceFile.FullPath); string tmpFileName = callback.TempFilesManager.GenerateNewName(); trace.Info("Temporary filename to download to: {0}", tmpFileName); Func <Stream, long, string, Task> writeToTempFile = async(fromStream, contentLength, description) => { using (FileStream fs = new FileStream(tmpFileName, FileMode.Create)) using (var progress = contentLength != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null) { await IOUtils.CopyStreamWithProgressAsync(fromStream, fs, downloadedBytes => { callback.SetStepDescription(string.Format("{2} {0}: {1}", IOUtils.FileSizeToString(downloadedBytes), sourceFile.FullPath, description)); if (progress != null) { progress.SetValue((double)downloadedBytes / (double)contentLength); } }, callback.Cancellation); } }; var uri = new Uri(sourceFile.Location); LogDownloaderRule logDownloaderRule; using (var cachedValue = cache.GetValue(uri)) { if (cachedValue != null) { await writeToTempFile(cachedValue, cachedValue.Length, "Loading from cache"); } else if ((logDownloaderRule = config.GetLogDownloaderConfig(uri)) != null && logDownloaderRule.UseWebBrowserDownloader) { using (var stream = await webBrowserDownloader.Download(new WebViewTools.DownloadParams() { Location = uri, ExpectedMimeType = logDownloaderRule.ExpectedMimeType, Cancellation = callback.Cancellation, Progress = progressAggregator, IsLoginUrl = testUri => logDownloaderRule.LoginUrls.Any(loginUrl => testUri.GetLeftPart(UriPartial.Path).Contains(loginUrl)) })) { await writeToTempFile(stream, 0, "Downloading"); } } else if (useHttpClient) { using (var client = new System.Net.Http.HttpClient()) { trace.Info("Start downloading {0}", sourceFile.Location); var response = await client.GetAsync(uri); await writeToTempFile(await response.Content.ReadAsStreamAsync(), response.Content.Headers.ContentLength.GetValueOrDefault(0), "Downloading"); } } else { using (WebClient client = new WebClient()) using (ManualResetEvent completed = new ManualResetEvent(false)) { ServicePointManager.ServerCertificateValidationCallback = delegate { return(true); }; var credentials = new CredentialsImpl() { Callback = callback, CredCache = credCache }; client.Credentials = credentials; Exception failure = null; client.OpenReadCompleted += (s, evt) => { failure = evt.Error; if (failure != null) { trace.Error(failure, "Downloading {0} completed with error", sourceFile.Location); } if (failure == null && (evt.Cancelled || callback.Cancellation.IsCancellationRequested)) { trace.Warning("Downloading {0} cancelled", sourceFile.Location); failure = new Exception("Aborted"); } if (failure == null) { try { long contentLength; long.TryParse(client.ResponseHeaders["Content-Length"] ?? "", out contentLength); writeToTempFile(evt.Result, contentLength, "Downloading").Wait(); } catch (Exception e) { trace.Error(e, "Failed saving to file"); failure = e; } } completed.Set(); }; trace.Info("Start downloading {0}", sourceFile.Location); client.OpenReadAsync(uri); if (WaitHandle.WaitAny(new WaitHandle[] { completed, callback.Cancellation.WaitHandle }) == 1) { trace.Info("Cancellation event was triggered. Cancelling download."); client.CancelAsync(); completed.WaitOne(); } HandleFailure(callback, credentials, failure); using (FileStream fs = new FileStream(tmpFileName, FileMode.Open)) { cache.SetValue(new Uri(sourceFile.Location), fs).Wait(); } } } } string preprocessingStep = name; return(new PreprocessingStepParams( tmpFileName, sourceFile.FullPath, sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(preprocessingStep)) )); } }
private void DoExtract( IPreprocessingStepCallback callback, string specificFileToExtract, Func <PreprocessingStepParams, bool> onNext, string password) { using (var zipFile = new Ionic.Zip.ZipFile(sourceFile.Uri)) { if (password != null) { zipFile.Password = password; } string currentEntryBeingExtracted = null; Progress.IProgressEventsSink progress = null; zipFile.ExtractProgress += (s, evt) => { evt.Cancel = callback.Cancellation.IsCancellationRequested; if (currentEntryBeingExtracted != null && evt.TotalBytesToTransfer != 0) { callback.SetStepDescription(string.Format("Unpacking {1}%: {0}", currentEntryBeingExtracted, evt.BytesTransferred * (long)100 / evt.TotalBytesToTransfer)); if (progress != null) { progress.SetValue( (double)evt.BytesTransferred / (double)evt.TotalBytesToTransfer); } } }; var entriesToEnum = specificFileToExtract != null? Enumerable.Repeat(zipFile[specificFileToExtract], 1) : zipFile.Entries; foreach (var entry in entriesToEnum.Where(e => e != null)) { if (entry.IsDirectory) { continue; } string entryFullPath = sourceFile.FullPath + "\\" + entry.FileName; string tmpFileName = callback.TempFilesManager.GenerateNewName(); callback.SetStepDescription("Unpacking " + entryFullPath); using (FileStream tmpFs = new FileStream(tmpFileName, FileMode.CreateNew)) using (var entryProgress = progressAggregator.CreateProgressSink()) { currentEntryBeingExtracted = entryFullPath; progress = entryProgress; entry.Extract(tmpFs); currentEntryBeingExtracted = null; progress = null; } string preprocessingStep = string.Format("{0} {1}", name, entry.FileName); if (!onNext(new PreprocessingStepParams(tmpFileName, entryFullPath, Utils.Concat(sourceFile.PreprocessingSteps, preprocessingStep)))) { break; } } } }