public override void ProcessSource( ISourceDataProcessor <CloudInitLogParsedEntry, LogContext, LogParsedDataKey> dataProcessor, ILogger logger, IProgress <int> progress, CancellationToken cancellationToken) { if (FilePaths.Length == 0) { return; } Timestamp startTime = Timestamp.MaxValue; Timestamp endTime = Timestamp.MinValue; DateTime fileStartTime = default(DateTime); long startNanoSeconds = 0; foreach (var path in FilePaths) { ulong lineNumber = 0; string line; var file = new StreamReader(path); var logEntries = new List <LogEntry>(); while ((line = file.ReadLine()) != null) { lineNumber++; var cloudInitMatch = CloudInitRegex.Match(line); if (cloudInitMatch.Success) { DateTime time; if (!DateTime.TryParse(cloudInitMatch.Groups[1].Value.Replace(",", "."), out time)) { throw new InvalidOperationException("Time cannot be parsed to DateTime format"); } var utcTime = DateTime.FromFileTimeUtc(time.ToFileTimeUtc()); // Need to explicitly say log time is in UTC, otherwise it will be interpreted as local var timeStamp = Timestamp.FromNanoseconds(utcTime.Ticks * 100); if (timeStamp < startTime) { startTime = timeStamp; fileStartTime = utcTime; startNanoSeconds = startTime.ToNanoseconds; } if (timeStamp > endTime) { endTime = timeStamp; } var offsetEventTimestamp = new Timestamp(timeStamp.ToNanoseconds - startNanoSeconds); LogEntry entry = new LogEntry(path, lineNumber, offsetEventTimestamp, cloudInitMatch.Groups[2].Value, cloudInitMatch.Groups[3].Value, cloudInitMatch.Groups[4].Value); dataProcessor.ProcessDataElement(entry, Context, cancellationToken); } else { Debugger.Break(); } } var offsetEndTimestamp = new Timestamp(endTime.ToNanoseconds - startNanoSeconds); this.dataSourceInfo = new DataSourceInfo(0, offsetEndTimestamp.ToNanoseconds, fileStartTime); Context.UpdateFileMetadata(path, new FileMetadata(lineNumber)); } }
public override void ProcessSource( ISourceDataProcessor <AndroidLogcatLogParsedEntry, LogContext, LogParsedDataKey> dataProcessor, ILogger logger, IProgress <int> progress, CancellationToken cancellationToken) { var contentDictionary = new Dictionary <string, IReadOnlyList <LogEntry> >(); Timestamp oldestTimestamp = new Timestamp(long.MaxValue); Timestamp newestTimestamp = new Timestamp(long.MinValue); long startNanoSeconds = 0; DateTime fileStartTime = default; var dateTimeCultureInfo = new CultureInfo("en-US"); double? utcOffsetInHours = null; var processingExceptions = new List <Exception>(); foreach (var path in FilePaths) { ulong currentLineNumber = 1; string line = string.Empty; try { var file = new System.IO.StreamReader(path); var logEntries = new List <LogEntry>(); var durationLogEntries = new List <DurationLogEntry>(); string timeDateFormat = null; var rootFolder = Path.GetDirectoryName(path); var utcOffsetFilePath = Path.Combine(rootFolder, "utcoffset.txt"); if (File.Exists(utcOffsetFilePath)) { var utcOffSetStr = File.ReadAllText(utcOffsetFilePath); if (double.TryParse(utcOffSetStr, out double utcOffsetTmp)) { utcOffsetInHours = utcOffsetTmp; } } while ((line = file.ReadLine()) != null) { LogEntry logEntry = null; // Optimization - don't save blank lines if (line == String.Empty) { currentLineNumber++; continue; } var androidLogCatMatch = AndroidLogCatRegex.Match(line); if (androidLogCatMatch.Success && timeDateFormat == null) { const string monthDayFormat = "MM-dd HH:mm:ss.fff"; const string monthDayYearFormat = "MM-dd-yyyy HH:mm:ss.fff"; if (DateTime.TryParseExact(androidLogCatMatch.Groups[1].Value, monthDayFormat, dateTimeCultureInfo, DateTimeStyles.None, out _)) { timeDateFormat = monthDayFormat; } else if (DateTime.TryParseExact(androidLogCatMatch.Groups[1].Value, monthDayYearFormat, dateTimeCultureInfo, DateTimeStyles.None, out _)) { timeDateFormat = monthDayYearFormat; } else { throw new Exception($"Invalid date/time format: {androidLogCatMatch.Groups[1].Value}"); } } // First, we check if the line is a new log entry if it matched Regex and by trying to parse its timestamp if (androidLogCatMatch.Success && androidLogCatMatch.Groups.Count >= 7 && DateTime.TryParseExact(androidLogCatMatch.Groups[1].Value, timeDateFormat, dateTimeCultureInfo, DateTimeStyles.None, out DateTime parsedTime)) { var timeStamp = Timestamp.FromNanoseconds(parsedTime.Ticks * 100); if (timeStamp < oldestTimestamp) { oldestTimestamp = timeStamp; fileStartTime = parsedTime; startNanoSeconds = oldestTimestamp.ToNanoseconds; } if (timeStamp > newestTimestamp) { newestTimestamp = timeStamp; } logEntry = new LogEntry { Timestamp = new Timestamp(timeStamp.ToNanoseconds), // We can't subtract startNanoSeconds here because logs are not necessarily time ordered FilePath = path, LineNumber = currentLineNumber, PID = uint.Parse(androidLogCatMatch.Groups[2].Value), TID = uint.Parse(androidLogCatMatch.Groups[3].Value), Priority = Utilities.Common.StringIntern(androidLogCatMatch.Groups[4].Value), Tag = Utilities.Common.StringIntern(androidLogCatMatch.Groups[5].Value.Trim()), Message = androidLogCatMatch.Groups[6].Value, }; // Specialized Duration parsing DurationLogEntry durationLogEntry = null; if (logEntry.Tag == "init" && logEntry.Message.Contains("took")) { var messageSplit = logEntry.Message.Split(); var firstSingleQuoteIdx = logEntry.Message.IndexOf('\''); var secondSingleQuoteIdx = logEntry.Message.IndexOf('\'', firstSingleQuoteIdx + 1); var name = logEntry.Message.Substring(firstSingleQuoteIdx + 1, secondSingleQuoteIdx - firstSingleQuoteIdx - 1); // Command 'write /dev/cpuctl/cpu.rt_period_us 1000000' action=init (/system/etc/init/hw/init.rc:271) took 0ms and failed: .... if (logEntry.Message.Contains("0ms")) { durationLogEntry = LogEntryFromDurationNs(0, logEntry, name); } // Service 'boringssl_self_test32_vendor' (pid 18) exited with status 0 waiting took 0.022000 seconds if (messageSplit[^ 3] == "took" && messageSplit[^ 1] == "seconds")
public override void ProcessSource( ISourceDataProcessor <DmesgIsoLogParsedEntry, LogContext, LogParsedDataKey> dataProcessor, ILogger logger, IProgress <int> progress, CancellationToken cancellationToken) { var contentDictionary = new Dictionary <string, IReadOnlyList <LogEntry> >(); string[] lineContent; string[] firstSlice; StringBuilder builder = new StringBuilder(); Timestamp oldestTimestamp = new Timestamp(long.MaxValue); Timestamp newestTImestamp = new Timestamp(long.MinValue); long startNanoSeconds = 0; DateTime fileStartTime = default(DateTime); DateTime parsedTime = default(DateTime); var dateTimeCultureInfo = new CultureInfo("en-US"); foreach (var path in FilePaths) { ulong currentLineNumber = 1; var file = new System.IO.StreamReader(path); string line = string.Empty; var entriesList = new List <LogEntry>(); LogEntry lastEntry = null; while ((line = file.ReadLine()) != null) { //First, we check if the line is a new log entry by trying to parse its timestamp if (line.Length >= 31 && DateTime.TryParseExact(line.Substring(0, 31), "yyyy-MM-ddTHH:mm:ss,ffffffK", dateTimeCultureInfo, DateTimeStyles.None, out parsedTime)) { if (lastEntry != null) { dataProcessor.ProcessDataElement(lastEntry, Context, cancellationToken); } lastEntry = new LogEntry(); lastEntry.filePath = path; lastEntry.lineNumber = currentLineNumber; lastEntry.rawLog = line.Substring(32); lineContent = lastEntry.rawLog.Split(':'); if (lineContent.Length == 1 || lineContent[0].Length > 20) { // Character ':' is not present in the beginning of the message, therefore there is no entity nor topic nor metadata lastEntry.entity = string.Empty; lastEntry.metadata = string.Empty; lastEntry.topic = string.Empty; lastEntry.message = lineContent[0]; } else if (lineContent.Length == 2) { // Character ':' occurs once in the message and in the beginning, therefore there is entity but no topic nor metadata lastEntry.entity = lineContent[0]; lastEntry.metadata = string.Empty; lastEntry.topic = string.Empty; lastEntry.message = lineContent[1]; } else { // Character ':' occurs multiple times in the message, and at least once in the beginning // We proceed to try to infer entity, metadata and topic firstSlice = lineContent[0].Split(' '); var lastSubstring = firstSlice[firstSlice.Length - 1]; int contentIndex = 2; if (firstSlice.Length > 1 && this.IsNumberFormat(lastSubstring) && this.IsNumberFormat(lineContent[1])) { //There is metadata and entity builder.Clear(); builder.Append(lastSubstring); builder.Append(':'); builder.Append(lineContent[1]); while (contentIndex < lineContent.Length && this.IsNumberFormat(lineContent[contentIndex])) { builder.Append(':'); builder.Append(lineContent[contentIndex]); ++contentIndex; } lastEntry.metadata = builder.ToString(); lastEntry.entity = lineContent[0].Substring(0, lineContent[0].Length - lastSubstring.Length - 1); if ((contentIndex < lineContent.Length - 1) && !IsPCIinfo(lineContent[contentIndex])) { //We check for topic after the metadata lastEntry.topic = lineContent[contentIndex]; ++contentIndex; } } else if (this.IsNumberFormat(lineContent[0])) { //There is metadata but no entity builder.Clear(); builder.Append(lineContent[0]); contentIndex = 1; while (contentIndex < lineContent.Length && this.IsNumberFormat(lineContent[contentIndex])) { builder.Append(':'); builder.Append(lineContent[contentIndex]); ++contentIndex; } lastEntry.metadata = builder.ToString(); lastEntry.entity = string.Empty; lastEntry.topic = string.Empty; } else { //There is entity but no metadata if (lineContent[1].StartsWith(" type=")) { //We check for topic after the entity lastEntry.topic = string.Empty; contentIndex = 1; } else { lastEntry.topic = lineContent[1]; } lastEntry.entity = lineContent[0]; lastEntry.metadata = string.Empty; } builder.Clear(); //Remainder of the log is the message if (contentIndex < lineContent.Length) { builder.Append(lineContent[contentIndex]); ++contentIndex; } while (contentIndex < lineContent.Length) { builder.Append(':'); builder.Append(lineContent[contentIndex]); ++contentIndex; } lastEntry.message = builder.ToString(); } parsedTime = DateTime.FromFileTimeUtc(parsedTime.ToFileTimeUtc()); // Need to explicitly say log time is in UTC, otherwise it will be interpreted as local var timeStamp = Timestamp.FromNanoseconds(parsedTime.Ticks * 100); if (timeStamp < oldestTimestamp) { oldestTimestamp = timeStamp; fileStartTime = parsedTime; startNanoSeconds = oldestTimestamp.ToNanoseconds; } if (timeStamp > newestTImestamp) { newestTImestamp = timeStamp; } lastEntry.timestamp = new Timestamp(timeStamp.ToNanoseconds - startNanoSeconds); entriesList.Add(lastEntry); } else if (entriesList.Count > 0) { if (lastEntry == null) { throw new InvalidOperationException("Can't parse the log."); } lastEntry.message = lastEntry.message + "\n" + line; lastEntry.rawLog = lastEntry.rawLog + "\n" + line; } ++currentLineNumber; } if (lastEntry != null) { dataProcessor.ProcessDataElement(lastEntry, Context, cancellationToken); } contentDictionary[path] = entriesList.AsReadOnly(); file.Close(); --currentLineNumber; Context.UpdateFileMetadata(path, new FileMetadata(currentLineNumber)); } var offsetEndTimestamp = new Timestamp(newestTImestamp.ToNanoseconds - startNanoSeconds); dataSourceInfo = new DataSourceInfo(0, offsetEndTimestamp.ToNanoseconds, fileStartTime); }
public override void ProcessSource( ISourceDataProcessor <WaLinuxAgentLogParsedEntry, LogContext, LogParsedDataKey> dataProcessor, ILogger logger, IProgress <int> progress, CancellationToken cancellationToken) { if (FilePaths.Length == 0) { return; } Timestamp startTime = Timestamp.MaxValue; Timestamp endTime = Timestamp.MinValue; DateTime fileStartTime = default(DateTime); long startNanoSeconds = 0; foreach (var path in FilePaths) { ulong lineNumber = 0; string line; var file = new StreamReader(path); var logEntries = new List <LogEntry>(); LogEntry lastLogEntry = null; while ((line = file.ReadLine()) != null) { lineNumber++; var WaLinuxAgentMatch = WaLinuxAgentRegex.Match(line); if (!WaLinuxAgentMatch.Success) { // Continuation of the last line. if (lastLogEntry == null) { throw new InvalidOperationException("Can't find the timestamp of the log."); } lastLogEntry.Log += '\n' + line; continue; } // We successfully match the format of the log, which means the last log entry is completed. if (lastLogEntry != null) { dataProcessor.ProcessDataElement(lastLogEntry, Context, cancellationToken); } DateTime time; if (!DateTime.TryParse(WaLinuxAgentMatch.Groups[1].Value, out time)) { throw new InvalidOperationException("Time cannot be parsed to DateTime format"); } var utcTime = DateTime.FromFileTimeUtc(time.ToFileTimeUtc()); // Need to explicitly say log time is in UTC, otherwise it will be interpreted as local var timeStamp = Timestamp.FromNanoseconds(utcTime.Ticks * 100); if (timeStamp < startTime) { startTime = timeStamp; fileStartTime = utcTime; startNanoSeconds = startTime.ToNanoseconds; } if (timeStamp > endTime) { endTime = timeStamp; } var offsetEventTimestamp = new Timestamp(timeStamp.ToNanoseconds - startNanoSeconds); lastLogEntry = new LogEntry(path, lineNumber, offsetEventTimestamp, WaLinuxAgentMatch.Groups[2].Value, WaLinuxAgentMatch.Groups[3].Value); } if (lastLogEntry != null) { dataProcessor.ProcessDataElement(lastLogEntry, Context, cancellationToken); } Context.UpdateFileMetadata(path, new FileMetadata(lineNumber)); } var offsetEndTimestamp = new Timestamp(endTime.ToNanoseconds - startNanoSeconds); this.dataSourceInfo = new DataSourceInfo(0, offsetEndTimestamp.ToNanoseconds, fileStartTime); }