public virtual void Run(string inputFile = null) { var cfg = Current; if (inputFile != null) { cfg.InputFile = inputFile; } var parser = cfg.Parser; BeginRunSummary(cfg.Parser.DateTimeFormat); var runSummary = RunSummary; // pre-filtering is done here var inputFiles = GatherInputFiles(); if (!inputFiles.Any()) { InvokeReportProgress("No log files found in input folder or none passed pre-filtering.", 100); return; } // NOTE: files are ordered here by LastWriteTime // reverse it to preserve the order in the output files foreach (var fileInfo in inputFiles.Reverse()) { try { Run(fileInfo); } catch (ParserException) { if (parser.NonStandardLines.Any()) { // write non-standard entries to runSummary runSummary.NonStandardEntries += parser.NonStandardLines.Count; var filePath = fileInfo.FullName; var outputPath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, $"[{cfg.ParserName}-FAILED]-"); if (FileProcessor.WriteFile(outputPath, parser.NonStandardLines, cfg.OverwriteFiles)) { runSummary.FilesWritten++; } } throw; } } if (inputFiles.Length > 1) { // do not write run summary if // only one file has been processed EndRunSummary(); } }
protected void Split(string filePath, LogEntry[] filteredEntries, Summary currentSummary) { var cfg = Current; var parser = cfg.Parser; var runSummary = RunSummary; if (cfg.SplitByThreads != null) { // whether or not we should write file for each thread var splitByAllThreads = cfg.SplitByThreads.Length == 0; var groupsByKey = filteredEntries.GroupBy(entry => entry.Thread); foreach (var groupedEntries in groupsByKey) { // if we're not splitting by all threads, other threads should be skipped if (!splitByAllThreads && !cfg.SplitByThreads.Contains(groupedEntries.Key)) { continue; } var currentThreadOutputPath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, $"[THREAD#{groupedEntries.Key}]-"); if (FileProcessor.WriteFile(currentThreadOutputPath, parser.ToLines(groupedEntries), cfg.OverwriteFiles)) { runSummary.FilesWritten++; currentSummary.FilesWritten++; } InvokeReportProgress($"\rTHREAD#{groupedEntries.Key}: {groupedEntries.Count()}", -1); } InvokeReportProgress(string.Empty); } if (cfg.SplitByIdentities != null) { // whether or not we should write file for each identity var splitByAllIdentities = cfg.SplitByIdentities.Length == 0; var groupsByKey = filteredEntries.GroupBy(entry => entry.Identity); foreach (var groupedEntries in groupsByKey) { // if we're not splitting by all users, other users should be skipped if (!splitByAllIdentities && !cfg.SplitByIdentities.Contains(groupedEntries.Key)) { continue; } var currentIdentityOutputPath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, $"[IDENTITY#{groupedEntries.Key}]-"); if (FileProcessor.WriteFile(currentIdentityOutputPath, parser.ToLines(groupedEntries), cfg.OverwriteFiles)) { runSummary.FilesWritten++; currentSummary.FilesWritten++; } InvokeReportProgress($"\rIDENTITY#{groupedEntries.Key}: {groupedEntries.Count()}", -1); } InvokeReportProgress(string.Empty); } if (cfg.SplitByLogLevels != null) { // whether or not we should write file for each log level var splitByAllLogLevels = cfg.SplitByLogLevels.Length == 0; var groupsByKey = filteredEntries.GroupBy(entry => entry.Level); foreach (var groupedEntries in groupsByKey) { // if we're not splitting by all log levels, other levels should be skipped if (!splitByAllLogLevels && !cfg.SplitByLogLevels.Contains(groupedEntries.Key)) { continue; } var currentLogLevelOutputPath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, $"[LEVEL#{groupedEntries.Key}]-"); if (FileProcessor.WriteFile(currentLogLevelOutputPath, parser.ToLines(groupedEntries), cfg.OverwriteFiles)) { runSummary.FilesWritten++; currentSummary.FilesWritten++; } InvokeReportProgress($"\rLEVEL#{groupedEntries.Key}: {groupedEntries.Count()}", -1); } InvokeReportProgress(string.Empty); } }
protected virtual void Run(FileInfo logFileInput) { var cfg = Current; var parser = cfg.Parser; var filters = cfg.Filters; var runSummary = RunSummary; var filePath = logFileInput.FullName; var currentSummary = parser.BeginSummary(); InvokeReportProgress($"Reading file '{filePath}'..."); void ProgressCallback(int percent) { InvokeReportProgress("Processing...", percent); } var logLines = FileProcessor.ReadLogLines(filePath, ProgressCallback, out var linesRead, parser.Expression); InvokeReportProgress("Done! ", 100); InvokeReportProgress(Environment.NewLine); if (!string.IsNullOrEmpty(cfg.FilePrefix)) { // if we're reparsing we need to replace the original file name (that's with a prefix) // with a one without a prefix, and prefix it accordingly during this parser run filePath = FileProcessor.ExtractFileName(filePath, cfg.FilePrefix); } var currentOutputPath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, cfg.FilePrefix); var currentDirectoryOutputPath = FileProcessor.GetFileDirectory(currentOutputPath); currentSummary.CopyConfiguration(cfg, filePath, currentDirectoryOutputPath); runSummary.FilesRead++; currentSummary.FilesRead++; runSummary.LinesRead += (ulong)linesRead; currentSummary.LinesRead = (ulong)linesRead; runSummary.LogsRead += (ulong)logLines.Length; currentSummary.LogsRead = (ulong)logLines.Length; InvokeReportProgress($"Lines: {linesRead}, Logs: {logLines.Length}, Constructing entries..."); var logEntries = parser.ToLogEntry(logLines).ToArray(); runSummary.NonStandardEntries += parser.NonStandardLines.Count; currentSummary.NonStandardEntries = parser.NonStandardLines.Count; if (parser.NonStandardLines.Any()) { // write failed log entries and continue with the parsing run (if it didn't throw an exception, threshold has not been exceeded) var outputPath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, $"[{cfg.ParserName}-FAILED]-"); if (FileProcessor.WriteFile(outputPath, parser.NonStandardLines, cfg.OverwriteFiles)) { runSummary.FilesWritten++; currentSummary.FilesWritten++; } } runSummary.EntriesConstructed += (ulong)logEntries.Length; currentSummary.EntriesConstructed = (ulong)logEntries.Length; InvokeReportProgress($"Logs: {logLines.Length}, Constructed: {logEntries.Length}, Filtering file..."); var filteredEntries = parser.FilterLogEntries(logEntries, ProgressCallback); InvokeReportProgress("Done! ", 100); InvokeReportProgress(Environment.NewLine); runSummary.FilteredEntries += (ulong)filteredEntries.Length; currentSummary.FilteredEntries = (ulong)filteredEntries.Length; InvokeReportProgress($"Entries: {logEntries.Length}, Filtered: {filteredEntries.Length}, Writing files..."); if (filteredEntries.Any()) { Split(filePath, filteredEntries, currentSummary); // write filtered file var filteredLines = parser.ToLines(filteredEntries).ToArray(); var filteredEntriesOutputPath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, "filtered"); if (FileProcessor.WriteFile(filteredEntriesOutputPath, filteredLines, cfg.OverwriteFiles)) { runSummary.FilesWritten++; currentSummary.FilesWritten++; runSummary.LinesWritten += (ulong)filteredLines.Length; currentSummary.LinesWritten = (ulong)filteredLines.Length; } InvokeReportProgress($"FILTERED: {filteredEntries.Length}"); } else { // TODO: At level WARN! InvokeReportProgress("No filtered entries resulted after run."); } // write entries accumulated in the filters var dedicatedFilterLogs = filters.Where(f => f.Type == FilterType.WriteToFile || f.Type == FilterType.IncludeAndWriteToFile); foreach (var dedicatedFilter in dedicatedFilterLogs) { // do not write empty files if (!dedicatedFilter.Entries.Any()) { continue; } // write one file per each custom filter that requires output var dedicatedFilterOutputPath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, dedicatedFilter.Name); //dedicatedFilter.FileName = dedicatedFilterOutputPath; // TODO: Figure out what's the idea here? if (FileProcessor.WriteFile(dedicatedFilterOutputPath, parser.ToLines(dedicatedFilter.Entries), cfg.OverwriteFiles)) { runSummary.FilesWritten++; currentSummary.FilesWritten++; } } if (cfg.CopyOriginal) { InvokeReportProgress("Writing original file..."); // write the original file to output folder var originalOutputFilePath = FileProcessor.GetOutputFilePath(filePath, cfg.InputFolder, cfg.OutputFolder, "original"); if (FileProcessor.WriteOriginalFile(sourcePath: filePath, destinationPath: originalOutputFilePath, lines: logLines, overwrite: cfg.OverwriteFiles)) { runSummary.FilesWritten++; currentSummary.FilesWritten++; } } // end current summary parser.EndSummary(); // write current summary var currentSummaryOutputPath = FileProcessor.GetSummaryFilePath(filePath, cfg.InputFolder, cfg.OutputFolder); if (FileProcessor.WriteFile(currentSummaryOutputPath, currentSummary.ToJson(), cfg.OverwriteFiles)) { FileProcessor.SetReadonly(currentSummaryOutputPath); runSummary.FilesWritten++; } AggregateRunSummaryCounters(currentSummary); // NOTE: A double empty console line intended InvokeReportProgress("Done!" + Environment.NewLine); }