Esempio n. 1
0
        async Task IPreprocessingStep.Execute(IPreprocessingStepCallback callback)
        {
            await callback.BecomeLongRunning();

            callback.SetStepDescription("Opening workspace " + source.FullPath);
            callback.SetOption(PreprocessingOptions.SkipLogsSelectionDialog, true);

            foreach (var entry in await await invoke.Invoke(() => workspacesManager.LoadWorkspace(source.Uri, callback.Cancellation), callback.Cancellation))
            {
                callback.YieldChildPreprocessing(entry.Log, entry.IsHiddenLog);
            }
        }
Esempio n. 2
0
        async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback, PreprocessingStepParams[] keyFiles)
        {
            await callback.BecomeLongRunning();

            callback.TempFilesCleanupList.Add(sourceFile.Location);
            callback.SetStepDescription("scanning...");

            string tmpFileName = callback.TempFilesManager.GenerateNewName();

            await Converters.PcapToPdmp(sourceFile.Location, keyFiles.Select(f => f.Location).ToArray(),
                                        tmpFileName, tshark, callback.Cancellation, callback.SetStepDescription, callback.Trace);

            return(new PreprocessingStepParams(
                       tmpFileName,
                       $"{sourceFile.FullPath}\\as_pdml",
                       sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName, StepArgument.ToString(keyFiles))),
                       $"{sourceFile.FullPath} (converted to PDML)"
                       ));
        }
Esempio n. 3
0
        async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
        {
            await callback.BecomeLongRunning();

            if (!TryParseUrl(source.Location, out var request))
            {
                throw new ArgumentException($"Can not parse URL {source.Location}");
            }

            using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() =>
            {
                var logs = await CloudWatchDownloader.Download(
                    webViewTools, request, callback.SetStepDescription);
                string zipTmpFileName = callback.TempFilesManager.GenerateNewName();
                using (var zipToOpen = new FileStream(zipTmpFileName, FileMode.CreateNew))
                    using (var archive = new ZipArchive(zipToOpen, ZipArchiveMode.Create))
                    {
                        foreach (var l in logs)
                        {
                            string tmpFile = callback.TempFilesManager.GenerateNewName();
                            File.WriteAllText(tmpFile, l.Value);
                            archive.CreateEntryFromFile(tmpFile, l.Key);
                            File.Delete(tmpFile);
                        }
                    }
                return(zipTmpFileName);
            }))
            {
                if (!sharedDownloadTask.IsValueCreator)
                {
                    callback.SetStepDescription("Waiting for downloaded data...");
                }
                var tmpFileName = await sharedDownloadTask.Value;
                return(new PreprocessingStepParams(
                           tmpFileName,
                           source.FullPath,
                           source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName))
                           ));
            }
            // todo: cache
        }
Esempio n. 4
0
        async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
        {
            await callback.BecomeLongRunning();

            if (!TryParseUrl(source.Location, out var request))
            {
                throw new ArgumentException($"Can not parse URL {source.Location}");
            }

            using (var sharedDownloadTask = callback.GetOrAddSharedValue($"{stepName}:{source.Location}", async() =>
            {
                string zipTmpFileName = callback.TempFilesManager.GenerateNewName();
                using (var zipStream = new FileStream(zipTmpFileName, FileMode.CreateNew))
                    using (var cachedStream = contentCache.GetValue(source.Location))
                    {
                        if (cachedStream != null)
                        {
                            await cachedStream.CopyToAsync(zipStream);
                        }
                        else
                        {
                            await DownloadAndMakeZip(request, zipStream, callback);
                            zipStream.Position = 0;
                            await contentCache.SetValue(source.Location, zipStream);
                        }
                    }
                return(zipTmpFileName);
            }))
            {
                if (!sharedDownloadTask.IsValueCreator)
                {
                    callback.SetStepDescription("Waiting for downloaded data...");
                }
                var tmpFileName = await sharedDownloadTask.Value;
                return(new PreprocessingStepParams(
                           tmpFileName,
                           source.FullPath,
                           source.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName))
                           ));
            }
        }
Esempio n. 5
0
        async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext)
        {
            await callback.BecomeLongRunning();

            callback.TempFilesCleanupList.Add(sourceFile.Uri);

            string tmpFileName = callback.TempFilesManager.GenerateNewName();

            await(new Writer()).Write(
                () => new FileStream(tmpFileName, FileMode.Create),
                s => s.Dispose(),
                FixTimestamps((new Reader(callback.Cancellation)).Read(
                                  sourceFile.Uri,
                                  progressHandler: prct => callback.SetStepDescription(
                                      string.Format("{0}: fixing timestamps {1}%", sourceFile.FullPath, (int)(prct * 100)))
                                  ))
                );

            onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\with_fixed_timestamps", sourceFile.FullPath),
                                               Utils.Concat(sourceFile.PreprocessingSteps, stepName)));
        }
Esempio n. 6
0
        static void AutodetectFormatAndYield(PreprocessingStepParams file, IPreprocessingStepCallback callback)
        {
            callback.SetStepDescription(string.Format("Detecting format: {0}", file.FullPath));
            var progressHandler = new ProgressHandler()
            {
                callback = callback
            };
            var detectedFormat = callback.FormatAutodetect.DetectFormat(file.Location, file.FullPath, progressHandler.cancellation.Token, progressHandler);

            if (detectedFormat != null)
            {
                file.DumpToConnectionParams(detectedFormat.ConnectParams);
                callback.YieldLogProvider(new YieldedProvider()
                {
                    Factory          = detectedFormat.Factory,
                    ConnectionParams = detectedFormat.ConnectParams,
                    DisplayName      = file.FullPath,
                    IsHiddenLog      = false
                });
            }
        }
Esempio n. 7
0
        async Task ExecuteInternal(IPreprocessingStepCallback callback, Action <PreprocessingStepParams> onNext)
        {
            await callback.BecomeLongRunning();

            callback.TempFilesCleanupList.Add(sourceFile.Location);

            string tmpFileName = callback.TempFilesManager.GenerateNewName();

            callback.SetStepDescription(string.Format("{0}: converting to text", sourceFile.FullPath));
            var harTask = Task.Run(() =>
            {
                Assembly dependencyResolveHandler(object s, ResolveEventArgs e)
                {
                    if (new AssemblyName(e.Name).Name == "Newtonsoft.Json")                     // HarConvert needs Newtonsoft.Json v6, map it to whatever modern version shipped with the plugin
                    {
                        return(typeof(Newtonsoft.Json.JsonReaderException).Assembly);
                    }
                    return(null);
                }
                AppDomain.CurrentDomain.AssemblyResolve += dependencyResolveHandler;
                try
                {
                    return(HarConvert.DeserializeFromFile(sourceFile.Location));
                }
                catch (Newtonsoft.Json.JsonReaderException)
                {
                    string fixedJsonFileName = callback.TempFilesManager.GenerateNewName();
                    try
                    {
                        TryFixJson(sourceFile.Location, fixedJsonFileName);
                        return(HarConvert.DeserializeFromFile(fixedJsonFileName));
                    }
                    catch (Newtonsoft.Json.JsonReaderException e)
                    {
                        throw new Exception(string.Format("HTTP archive is broken"), e);
                    }
                    finally
                    {
                        if (File.Exists(fixedJsonFileName))
                        {
                            File.Delete(fixedJsonFileName);
                        }
                    }
                }
                finally
                {
                    AppDomain.CurrentDomain.AssemblyResolve -= dependencyResolveHandler;
                }
            });

            if (await Task.WhenAny(ToTask(callback.Cancellation), harTask) != harTask)
            {
                return;
            }

            await(new Writer()).Write(
                () => new FileStream(tmpFileName, FileMode.Create),
                s => s.Dispose(),
                ToText(harTask.Result, callback.Cancellation)
                );

            onNext(new PreprocessingStepParams(tmpFileName, string.Format("{0}\\text", sourceFile.FullPath),
                                               sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(stepName)), sourceFile.FullPath));
        }
Esempio n. 8
0
        private async Task DoExtract(
            IPreprocessingStepCallback callback,
            string specificFileToExtract,
            Func <PreprocessingStepParams, bool> onNext,
            string password,
            IFileSystem fileSystem)
        {
            async Task <ICSharpCode.SharpZipLib.Zip.ZipFile> CreateZipFile()
            {
                if (!IsBrowser.Value)
                {
                    return(new ICSharpCode.SharpZipLib.Zip.ZipFile(@params.Location));
                }
                else
                {
                    var ms = new MemoryStream();
                    using (var fileStream = await fileSystem.OpenFile(@params.Location))
                        await IOUtils.CopyStreamWithProgressAsync(fileStream, ms, _ => { }, callback.Cancellation);
                    return(new ICSharpCode.SharpZipLib.Zip.ZipFile(ms, leaveOpen: false));
                }
            }

            using (var zipFile = await CreateZipFile())
            {
                if (password != null)
                {
                    zipFile.Password = password;
                }
                var entriesToEnum = specificFileToExtract != null?
                                    Enumerable.Repeat(zipFile.GetEntry(specificFileToExtract), 1) : zipFile.OfType <ICSharpCode.SharpZipLib.Zip.ZipEntry>();

                foreach (var entry in entriesToEnum.Where(e => e != null))
                {
                    if (entry.IsDirectory)
                    {
                        continue;
                    }

                    if (entry.IsCrypted && password == null)
                    {
                        throw new PasswordException();
                    }

                    string entryFullPath = @params.FullPath + "\\" + entry.Name;
                    string tmpFileName   = callback.TempFilesManager.GenerateNewName();

                    callback.SetStepDescription("Unpacking " + entryFullPath);
                    using (FileStream tmpFs = new FileStream(tmpFileName, FileMode.CreateNew))
                        using (var entryProgress = progressAggregator.CreateProgressSink())
                        {
                            using (var entryStream = zipFile.GetInputStream(entry))
                            {
                                var totalLen = entry.Size;
                                IOUtils.CopyStreamWithProgress(entryStream, tmpFs, pos =>
                                {
                                    if (totalLen > 0)
                                    {
                                        callback.SetStepDescription($"Unpacking {pos * 100 / totalLen}%: {entryFullPath}");
                                        entryProgress.SetValue((double)pos / totalLen);
                                    }
                                }, callback.Cancellation);
                            }
                        }

                    if (!onNext(new PreprocessingStepParams(tmpFileName, entryFullPath,
                                                            @params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, entry.Name)))))
                    {
                        break;
                    }
                }
            }
        }
Esempio n. 9
0
        async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
        {
            await callback.BecomeLongRunning();

            string factoryName = @params.Argument;

            callback.TempFilesCleanupList.Add(@params.Location);
            Action <double?> setStepDescription = prctComplete =>
            {
                var str = new StringBuilder();
                str.Append(@params.FullPath);
                str.Append(": fixing timestamp anomalies...");
                if (prctComplete != null)
                {
                    str.AppendFormat(" {0}%", (int)(prctComplete.Value * 100));
                }
                callback.SetStepDescription(str.ToString());
            };

            setStepDescription(null);

            string tmpFileName = callback.TempFilesManager.GenerateNewName();

            var factoryNameSplit = factoryName.Split('\\');

            if (factoryNameSplit.Length != 2)
            {
                throw new InvalidFormatException();
            }
            var factory = logProviderFactoryRegistry.Find(factoryNameSplit[0], factoryNameSplit[1]);

            if (factory == null)
            {
                throw new InvalidDataException("factory not found: " + factoryName);
            }
            var readerFactory = factory as IMediaBasedReaderFactory;

            if (readerFactory == null)
            {
                throw new InvalidDataException("bad factory: " + factoryName);
            }
            using (ILogMedia fileMedia = await SimpleFileMedia.Create(fileSystem,
                                                                      SimpleFileMedia.CreateConnectionParamsFromFileName(@params.Location)))
                using (ILogSourceThreadsInternal threads = new LogSourceThreads())
                    using (var reader = readerFactory.CreateMessagesReader(
                               new MediaBasedReaderParams(threads, fileMedia)))
                    {
                        var readerImpl = reader as MediaBasedPositionedMessagesReader;         // todo: do not use real classes; have stream encoding in an interface.
                        if (readerImpl == null)
                        {
                            throw new InvalidDataException("bad reader was made by factory " + factoryName);
                        }
                        await reader.UpdateAvailableBounds(false);

                        var    range    = new FileRange.Range(reader.BeginPosition, reader.EndPosition);
                        double rangeLen = range.Length;
                        using (var progress = progressAggregator.CreateProgressSink())
                            using (var writer = new StreamWriter(tmpFileName, false, readerImpl.StreamEncoding))
                                await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition,
                                                                                                             flags: MessagesParserFlag.DisableDejitter | MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading)), async parser =>
                                {
                                    var queue = new VCSKicksCollection.PriorityQueue <IMessage>(
                                        new MessagesComparer(ignoreConnectionIds: true));
                                    Action dequeue          = () => writer.WriteLine(queue.Dequeue().RawText.ToString());
                                    double lastPrctComplete = 0;
                                    var cancellation        = callback.Cancellation;
                                    for (long msgIdx = 0;; ++msgIdx)
                                    {
                                        if (cancellation.IsCancellationRequested)
                                        {
                                            break;
                                        }
                                        var msg = await parser.ReadNext();
                                        if (msg == null)
                                        {
                                            break;
                                        }
                                        if ((msgIdx % progressUpdateThreshold) == 0 && rangeLen > 0)
                                        {
                                            var prctComplete = (double)(msg.Position - range.Begin) / rangeLen;
                                            progress.SetValue(prctComplete);
                                            if (prctComplete - lastPrctComplete > 0.05)
                                            {
                                                setStepDescription(prctComplete);
                                                lastPrctComplete = prctComplete;
                                            }
                                        }
                                        queue.Enqueue(msg);
                                        if (queue.Count > queueSize)
                                        {
                                            dequeue();
                                        }
                                    }
                                    while (queue.Count > 0)
                                    {
                                        dequeue();
                                    }
                                });
                    }

            return(new PreprocessingStepParams(
                       tmpFileName,
                       @params.FullPath + " (reordered)",
                       @params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, factoryName))
                       ));
        }
Esempio n. 10
0
        async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
        {
            var trace = callback.Trace;

            using (trace.NewFrame)
            {
                await callback.BecomeLongRunning();

                trace.Info("Downloading '{0}' from '{1}'", sourceFile.FullPath, sourceFile.Location);
                callback.SetStepDescription("Downloading " + sourceFile.FullPath);

                string tmpFileName = callback.TempFilesManager.GenerateNewName();
                trace.Info("Temporary filename to download to: {0}", tmpFileName);

                Func <Stream, long, string, Task> writeToTempFile = async(fromStream, contentLength, description) =>
                {
                    using (FileStream fs = new FileStream(tmpFileName, FileMode.Create))
                        using (var progress = contentLength != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
                        {
                            await IOUtils.CopyStreamWithProgressAsync(fromStream, fs, downloadedBytes =>
                            {
                                callback.SetStepDescription(string.Format("{2} {0}: {1}",
                                                                          IOUtils.FileSizeToString(downloadedBytes), sourceFile.FullPath, description));
                                if (progress != null)
                                {
                                    progress.SetValue((double)downloadedBytes / (double)contentLength);
                                }
                            }, callback.Cancellation);
                        }
                };

                var uri = new Uri(sourceFile.Location);
                LogDownloaderRule logDownloaderRule;
                using (var cachedValue = cache.GetValue(uri))
                {
                    if (cachedValue != null)
                    {
                        await writeToTempFile(cachedValue, cachedValue.Length, "Loading from cache");
                    }
                    else if ((logDownloaderRule = config.GetLogDownloaderConfig(uri)) != null && logDownloaderRule.UseWebBrowserDownloader)
                    {
                        using (var stream = await webBrowserDownloader.Download(new WebViewTools.DownloadParams()
                        {
                            Location = uri,
                            ExpectedMimeType = logDownloaderRule.ExpectedMimeType,
                            Cancellation = callback.Cancellation,
                            Progress = progressAggregator,
                            IsLoginUrl = testUri => logDownloaderRule.LoginUrls.Any(loginUrl => testUri.GetLeftPart(UriPartial.Path).Contains(loginUrl))
                        }))
                        {
                            await writeToTempFile(stream, 0, "Downloading");
                        }
                    }
                    else if (useHttpClient)
                    {
                        using (var client = new System.Net.Http.HttpClient())
                        {
                            trace.Info("Start downloading {0}", sourceFile.Location);
                            var response = await client.GetAsync(uri);
                            await writeToTempFile(await response.Content.ReadAsStreamAsync(),
                                                  response.Content.Headers.ContentLength.GetValueOrDefault(0), "Downloading");
                        }
                    }
                    else
                    {
                        using (WebClient client = new WebClient())
                            using (ManualResetEvent completed = new ManualResetEvent(false))
                            {
                                ServicePointManager.ServerCertificateValidationCallback = delegate { return(true); };

                                var credentials = new CredentialsImpl()
                                {
                                    Callback = callback, CredCache = credCache
                                };
                                client.Credentials = credentials;

                                Exception failure = null;
                                client.OpenReadCompleted += (s, evt) =>
                                {
                                    failure = evt.Error;
                                    if (failure != null)
                                    {
                                        trace.Error(failure, "Downloading {0} completed with error", sourceFile.Location);
                                    }
                                    if (failure == null && (evt.Cancelled || callback.Cancellation.IsCancellationRequested))
                                    {
                                        trace.Warning("Downloading {0} cancelled", sourceFile.Location);
                                        failure = new Exception("Aborted");
                                    }
                                    if (failure == null)
                                    {
                                        try
                                        {
                                            long contentLength;
                                            long.TryParse(client.ResponseHeaders["Content-Length"] ?? "", out contentLength);
                                            writeToTempFile(evt.Result, contentLength, "Downloading").Wait();
                                        }
                                        catch (Exception e)
                                        {
                                            trace.Error(e, "Failed saving to file");
                                            failure = e;
                                        }
                                    }
                                    completed.Set();
                                };

                                trace.Info("Start downloading {0}", sourceFile.Location);
                                client.OpenReadAsync(uri);

                                if (WaitHandle.WaitAny(new WaitHandle[] { completed, callback.Cancellation.WaitHandle }) == 1)
                                {
                                    trace.Info("Cancellation event was triggered. Cancelling download.");
                                    client.CancelAsync();
                                    completed.WaitOne();
                                }

                                HandleFailure(callback, credentials, failure);

                                using (FileStream fs = new FileStream(tmpFileName, FileMode.Open))
                                {
                                    cache.SetValue(new Uri(sourceFile.Location), fs).Wait();
                                }
                            }
                    }
                }

                string preprocessingStep = name;

                return(new PreprocessingStepParams(
                           tmpFileName, sourceFile.FullPath,
                           sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(preprocessingStep))
                           ));
            }
        }
Esempio n. 11
0
        private void DoExtract(
            IPreprocessingStepCallback callback,
            string specificFileToExtract,
            Func <PreprocessingStepParams, bool> onNext,
            string password)
        {
            using (var zipFile = new Ionic.Zip.ZipFile(sourceFile.Uri))
            {
                if (password != null)
                {
                    zipFile.Password = password;
                }
                string currentEntryBeingExtracted     = null;
                Progress.IProgressEventsSink progress = null;
                zipFile.ExtractProgress += (s, evt) =>
                {
                    evt.Cancel = callback.Cancellation.IsCancellationRequested;
                    if (currentEntryBeingExtracted != null && evt.TotalBytesToTransfer != 0)
                    {
                        callback.SetStepDescription(string.Format("Unpacking {1}%: {0}",
                                                                  currentEntryBeingExtracted,
                                                                  evt.BytesTransferred * (long)100 / evt.TotalBytesToTransfer));
                        if (progress != null)
                        {
                            progress.SetValue(
                                (double)evt.BytesTransferred / (double)evt.TotalBytesToTransfer);
                        }
                    }
                };
                var entriesToEnum = specificFileToExtract != null?
                                    Enumerable.Repeat(zipFile[specificFileToExtract], 1) : zipFile.Entries;

                foreach (var entry in entriesToEnum.Where(e => e != null))
                {
                    if (entry.IsDirectory)
                    {
                        continue;
                    }

                    string entryFullPath = sourceFile.FullPath + "\\" + entry.FileName;
                    string tmpFileName   = callback.TempFilesManager.GenerateNewName();

                    callback.SetStepDescription("Unpacking " + entryFullPath);
                    using (FileStream tmpFs = new FileStream(tmpFileName, FileMode.CreateNew))
                        using (var entryProgress = progressAggregator.CreateProgressSink())
                        {
                            currentEntryBeingExtracted = entryFullPath;
                            progress = entryProgress;
                            entry.Extract(tmpFs);
                            currentEntryBeingExtracted = null;
                            progress = null;
                        }

                    string preprocessingStep = string.Format("{0} {1}", name, entry.FileName);

                    if (!onNext(new PreprocessingStepParams(tmpFileName, entryFullPath,
                                                            Utils.Concat(sourceFile.PreprocessingSteps, preprocessingStep))))
                    {
                        break;
                    }
                }
            }
        }