private void Check() { var cfg = Current; if (string.IsNullOrWhiteSpace(cfg.InputFolder) && string.IsNullOrEmpty(cfg.InputFile)) { throw new ConfigurationException("No input, please specify either input file or folder."); } if (string.IsNullOrEmpty(cfg.OutputFolder)) { if (!string.IsNullOrWhiteSpace(cfg.InputFolder)) { var outputPath = $"{cfg.InputFolder}\\parsed\\"; cfg.OutputFolder = outputPath; } else if (!string.IsNullOrWhiteSpace(cfg.InputFile)) { var fileDirectory = FileProcessor.GetFileDirectory(cfg.InputFile); var outputPath = $"{fileDirectory}\\parsed\\"; cfg.OutputFolder = outputPath; } } if (cfg.Filters == null || !cfg.Filters.Any()) { throw new ConfigurationException("No filters provided, please provide at least one filter."); } Current.Parser = InstantiateParser(cfg.ParserName); }
protected virtual void Run(FileInfo logFileInput) { var cfg = Current; var parser = cfg.Parser; var filters = cfg.Filters; var runSummary = RunSummary; var filePath = logFileInput.FullName; var currentSummary = parser.BeginSummary(); InvokeReportProgress($"Reading file '{filePath}'..."); void ProgressCallback(int percent) { InvokeReportProgress("Processing...", percent); } var logLines = FileProcessor.ReadLogLines(filePath, ProgressCallback, out var linesRead, parser.Expression); InvokeReportProgress("Done! ", 100); InvokeReportProgress(Environment.NewLine); if (!string.IsNullOrEmpty(cfg.FilePrefix)) { // if we're reparsing we need to replace the original file name (that's with a prefix) // with a one without a prefix, and prefix it accordingly during this parser run filePath = FileProcessor.ExtractFileName(filePath, cfg.FilePrefix); } var currentOutputPath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, cfg.FilePrefix); var currentDirectoryOutputPath = FileProcessor.GetFileDirectory(currentOutputPath); currentSummary.CopyConfiguration(cfg, filePath, currentDirectoryOutputPath); runSummary.FilesRead++; currentSummary.FilesRead++; runSummary.LinesRead += (ulong)linesRead; currentSummary.LinesRead = (ulong)linesRead; runSummary.LogsRead += (ulong)logLines.Length; currentSummary.LogsRead = (ulong)logLines.Length; InvokeReportProgress($"Lines: {linesRead}, Logs: {logLines.Length}, Constructing entries..."); var logEntries = parser.ToLogEntry(logLines).ToArray(); runSummary.NonStandardEntries += parser.NonStandardLines.Count; currentSummary.NonStandardEntries = parser.NonStandardLines.Count; if (parser.NonStandardLines.Any()) { // write failed log entries and continue with the parsing run (if it didn't throw an exception, threshold has not been exceeded) var outputPath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, $"[{cfg.ParserName}-FAILED]-"); if (FileProcessor.WriteFile(outputPath, parser.NonStandardLines, cfg.OverwriteFiles)) { runSummary.FilesWritten++; currentSummary.FilesWritten++; } } runSummary.EntriesConstructed += (ulong)logEntries.Length; currentSummary.EntriesConstructed = (ulong)logEntries.Length; InvokeReportProgress($"Logs: {logLines.Length}, Constructed: {logEntries.Length}, Filtering file..."); var filteredEntries = parser.FilterLogEntries(logEntries, ProgressCallback); InvokeReportProgress("Done! ", 100); InvokeReportProgress(Environment.NewLine); runSummary.FilteredEntries += (ulong)filteredEntries.Length; currentSummary.FilteredEntries = (ulong)filteredEntries.Length; InvokeReportProgress($"Entries: {logEntries.Length}, Filtered: {filteredEntries.Length}, Writing files..."); if (filteredEntries.Any()) { Split(filePath, filteredEntries, currentSummary); // write filtered file var filteredLines = parser.ToLines(filteredEntries).ToArray(); var filteredEntriesOutputPath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, "filtered"); if (FileProcessor.WriteFile(filteredEntriesOutputPath, filteredLines, cfg.OverwriteFiles)) { runSummary.FilesWritten++; currentSummary.FilesWritten++; runSummary.LinesWritten += (ulong)filteredLines.Length; currentSummary.LinesWritten = (ulong)filteredLines.Length; } InvokeReportProgress($"FILTERED: {filteredEntries.Length}"); } else { // TODO: At level WARN! InvokeReportProgress("No filtered entries resulted after run."); } // write entries accumulated in the filters var dedicatedFilterLogs = filters.Where(f => f.Type == FilterType.WriteToFile || f.Type == FilterType.IncludeAndWriteToFile); foreach (var dedicatedFilter in dedicatedFilterLogs) { // do not write empty files if (!dedicatedFilter.Entries.Any()) { continue; } // write one file per each custom filter that requires output var dedicatedFilterOutputPath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, dedicatedFilter.Name); //dedicatedFilter.FileName = dedicatedFilterOutputPath; // TODO: Figure out what's the idea here? if (FileProcessor.WriteFile(dedicatedFilterOutputPath, parser.ToLines(dedicatedFilter.Entries), cfg.OverwriteFiles)) { runSummary.FilesWritten++; currentSummary.FilesWritten++; } } if (cfg.CopyOriginal) { InvokeReportProgress("Writing original file..."); // write the original file to output folder var originalOutputFilePath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, "original"); if (FileProcessor.WriteOriginalFile(sourcePath: filePath, destinationPath: originalOutputFilePath, lines: logLines, overwrite: cfg.OverwriteFiles)) { runSummary.FilesWritten++; currentSummary.FilesWritten++; } } // end current summary parser.EndSummary(); // write current summary var currentSummaryOutputPath = FileProcessor.GetSummaryFilePath(filePath, cfg.InputFolder, cfg.OutputFolder); if (FileProcessor.WriteFile(currentSummaryOutputPath, currentSummary.ToJson(), cfg.OverwriteFiles)) { FileProcessor.SetReadonly(currentSummaryOutputPath); runSummary.FilesWritten++; } AggregateRunSummaryCounters(currentSummary); // NOTE: A double empty console line intended InvokeReportProgress("Done!" + Environment.NewLine); }