public override void ProcessSource(
ISourceDataProcessor <CloudInitLogParsedEntry, LogContext, LogParsedDataKey> dataProcessor,
ILogger logger, IProgress <int> progress, CancellationToken cancellationToken)
{
if (FilePaths.Length == 0)
{
return;
}
Timestamp startTime = Timestamp.MaxValue;
Timestamp endTime = Timestamp.MinValue;
DateTime fileStartTime = default(DateTime);
long startNanoSeconds = 0;
foreach (var path in FilePaths)
{
ulong lineNumber = 0;
string line;
var file = new StreamReader(path);
var logEntries = new List <LogEntry>();
while ((line = file.ReadLine()) != null)
{
lineNumber++;
var cloudInitMatch = CloudInitRegex.Match(line);
if (cloudInitMatch.Success)
{
DateTime time;
if (!DateTime.TryParse(cloudInitMatch.Groups[1].Value.Replace(",", "."), out time))
{
throw new InvalidOperationException("Time cannot be parsed to DateTime format");
}
var utcTime = DateTime.FromFileTimeUtc(time.ToFileTimeUtc()); // Need to explicitly say log time is in UTC, otherwise it will be interpreted as local
var timeStamp = Timestamp.FromNanoseconds(utcTime.Ticks * 100);
if (timeStamp < startTime)
{
startTime = timeStamp;
fileStartTime = utcTime;
startNanoSeconds = startTime.ToNanoseconds;
}
if (timeStamp > endTime)
{
endTime = timeStamp;
}
var offsetEventTimestamp = new Timestamp(timeStamp.ToNanoseconds - startNanoSeconds);
LogEntry entry = new LogEntry(path, lineNumber, offsetEventTimestamp, cloudInitMatch.Groups[2].Value,
cloudInitMatch.Groups[3].Value, cloudInitMatch.Groups[4].Value);
dataProcessor.ProcessDataElement(entry, Context, cancellationToken);
}
else
{
Debugger.Break();
}
}
var offsetEndTimestamp = new Timestamp(endTime.ToNanoseconds - startNanoSeconds);
this.dataSourceInfo = new DataSourceInfo(0, offsetEndTimestamp.ToNanoseconds, fileStartTime);
Context.UpdateFileMetadata(path, new FileMetadata(lineNumber));
}
}
public override void ProcessSource(
ISourceDataProcessor <AndroidLogcatLogParsedEntry, LogContext, LogParsedDataKey> dataProcessor,
ILogger logger, IProgress <int> progress, CancellationToken cancellationToken)
{
var contentDictionary = new Dictionary <string, IReadOnlyList <LogEntry> >();
Timestamp oldestTimestamp = new Timestamp(long.MaxValue);
Timestamp newestTimestamp = new Timestamp(long.MinValue);
long startNanoSeconds = 0;
DateTime fileStartTime = default;
var dateTimeCultureInfo = new CultureInfo("en-US");
double? utcOffsetInHours = null;
var processingExceptions = new List <Exception>();
foreach (var path in FilePaths)
{
ulong currentLineNumber = 1;
string line = string.Empty;
try
{
var file = new System.IO.StreamReader(path);
var logEntries = new List <LogEntry>();
var durationLogEntries = new List <DurationLogEntry>();
string timeDateFormat = null;
var rootFolder = Path.GetDirectoryName(path);
var utcOffsetFilePath = Path.Combine(rootFolder, "utcoffset.txt");
if (File.Exists(utcOffsetFilePath))
{
var utcOffSetStr = File.ReadAllText(utcOffsetFilePath);
if (double.TryParse(utcOffSetStr, out double utcOffsetTmp))
{
utcOffsetInHours = utcOffsetTmp;
}
}
while ((line = file.ReadLine()) != null)
{
LogEntry logEntry = null;
// Optimization - don't save blank lines
if (line == String.Empty)
{
currentLineNumber++;
continue;
}
var androidLogCatMatch = AndroidLogCatRegex.Match(line);
if (androidLogCatMatch.Success && timeDateFormat == null)
{
const string monthDayFormat = "MM-dd HH:mm:ss.fff";
const string monthDayYearFormat = "MM-dd-yyyy HH:mm:ss.fff";
if (DateTime.TryParseExact(androidLogCatMatch.Groups[1].Value, monthDayFormat, dateTimeCultureInfo, DateTimeStyles.None, out _))
{
timeDateFormat = monthDayFormat;
}
else if (DateTime.TryParseExact(androidLogCatMatch.Groups[1].Value, monthDayYearFormat, dateTimeCultureInfo, DateTimeStyles.None, out _))
{
timeDateFormat = monthDayYearFormat;
}
else
{
throw new Exception($"Invalid date/time format: {androidLogCatMatch.Groups[1].Value}");
}
}
// First, we check if the line is a new log entry if it matched Regex and by trying to parse its timestamp
if (androidLogCatMatch.Success &&
androidLogCatMatch.Groups.Count >= 7 &&
DateTime.TryParseExact(androidLogCatMatch.Groups[1].Value, timeDateFormat, dateTimeCultureInfo, DateTimeStyles.None, out DateTime parsedTime))
{
var timeStamp = Timestamp.FromNanoseconds(parsedTime.Ticks * 100);
if (timeStamp < oldestTimestamp)
{
oldestTimestamp = timeStamp;
fileStartTime = parsedTime;
startNanoSeconds = oldestTimestamp.ToNanoseconds;
}
if (timeStamp > newestTimestamp)
{
newestTimestamp = timeStamp;
}
logEntry = new LogEntry
{
Timestamp = new Timestamp(timeStamp.ToNanoseconds), // We can't subtract startNanoSeconds here because logs are not necessarily time ordered
FilePath = path,
LineNumber = currentLineNumber,
PID = uint.Parse(androidLogCatMatch.Groups[2].Value),
TID = uint.Parse(androidLogCatMatch.Groups[3].Value),
Priority = Utilities.Common.StringIntern(androidLogCatMatch.Groups[4].Value),
Tag = Utilities.Common.StringIntern(androidLogCatMatch.Groups[5].Value.Trim()),
Message = androidLogCatMatch.Groups[6].Value,
};
// Specialized Duration parsing
DurationLogEntry durationLogEntry = null;
if (logEntry.Tag == "init" && logEntry.Message.Contains("took"))
{
var messageSplit = logEntry.Message.Split();
var firstSingleQuoteIdx = logEntry.Message.IndexOf('\'');
var secondSingleQuoteIdx = logEntry.Message.IndexOf('\'', firstSingleQuoteIdx + 1);
var name = logEntry.Message.Substring(firstSingleQuoteIdx + 1, secondSingleQuoteIdx - firstSingleQuoteIdx - 1);
// Command 'write /dev/cpuctl/cpu.rt_period_us 1000000' action=init (/system/etc/init/hw/init.rc:271) took 0ms and failed: ....
if (logEntry.Message.Contains("0ms"))
{
durationLogEntry = LogEntryFromDurationNs(0, logEntry, name);
}
// Service 'boringssl_self_test32_vendor' (pid 18) exited with status 0 waiting took 0.022000 seconds
if (messageSplit[^ 3] == "took" && messageSplit[^ 1] == "seconds")
public override void ProcessSource(
ISourceDataProcessor <DmesgIsoLogParsedEntry, LogContext, LogParsedDataKey> dataProcessor,
ILogger logger, IProgress <int> progress, CancellationToken cancellationToken)
{
var contentDictionary = new Dictionary <string, IReadOnlyList <LogEntry> >();
string[] lineContent;
string[] firstSlice;
StringBuilder builder = new StringBuilder();
Timestamp oldestTimestamp = new Timestamp(long.MaxValue);
Timestamp newestTImestamp = new Timestamp(long.MinValue);
long startNanoSeconds = 0;
DateTime fileStartTime = default(DateTime);
DateTime parsedTime = default(DateTime);
var dateTimeCultureInfo = new CultureInfo("en-US");
foreach (var path in FilePaths)
{
ulong currentLineNumber = 1;
var file = new System.IO.StreamReader(path);
string line = string.Empty;
var entriesList = new List <LogEntry>();
LogEntry lastEntry = null;
while ((line = file.ReadLine()) != null)
{
//First, we check if the line is a new log entry by trying to parse its timestamp
if (line.Length >= 31 && DateTime.TryParseExact(line.Substring(0, 31), "yyyy-MM-ddTHH:mm:ss,ffffffK", dateTimeCultureInfo, DateTimeStyles.None, out parsedTime))
{
if (lastEntry != null)
{
dataProcessor.ProcessDataElement(lastEntry, Context, cancellationToken);
}
lastEntry = new LogEntry();
lastEntry.filePath = path;
lastEntry.lineNumber = currentLineNumber;
lastEntry.rawLog = line.Substring(32);
lineContent = lastEntry.rawLog.Split(':');
if (lineContent.Length == 1 || lineContent[0].Length > 20)
{
// Character ':' is not present in the beginning of the message, therefore there is no entity nor topic nor metadata
lastEntry.entity = string.Empty;
lastEntry.metadata = string.Empty;
lastEntry.topic = string.Empty;
lastEntry.message = lineContent[0];
}
else if (lineContent.Length == 2)
{
// Character ':' occurs once in the message and in the beginning, therefore there is entity but no topic nor metadata
lastEntry.entity = lineContent[0];
lastEntry.metadata = string.Empty;
lastEntry.topic = string.Empty;
lastEntry.message = lineContent[1];
}
else
{
// Character ':' occurs multiple times in the message, and at least once in the beginning
// We proceed to try to infer entity, metadata and topic
firstSlice = lineContent[0].Split(' ');
var lastSubstring = firstSlice[firstSlice.Length - 1];
int contentIndex = 2;
if (firstSlice.Length > 1 && this.IsNumberFormat(lastSubstring) && this.IsNumberFormat(lineContent[1]))
{
//There is metadata and entity
builder.Clear();
builder.Append(lastSubstring);
builder.Append(':');
builder.Append(lineContent[1]);
while (contentIndex < lineContent.Length && this.IsNumberFormat(lineContent[contentIndex]))
{
builder.Append(':');
builder.Append(lineContent[contentIndex]);
++contentIndex;
}
lastEntry.metadata = builder.ToString();
lastEntry.entity = lineContent[0].Substring(0, lineContent[0].Length - lastSubstring.Length - 1);
if ((contentIndex < lineContent.Length - 1) && !IsPCIinfo(lineContent[contentIndex]))
{
//We check for topic after the metadata
lastEntry.topic = lineContent[contentIndex];
++contentIndex;
}
}
else if (this.IsNumberFormat(lineContent[0]))
{
//There is metadata but no entity
builder.Clear();
builder.Append(lineContent[0]);
contentIndex = 1;
while (contentIndex < lineContent.Length && this.IsNumberFormat(lineContent[contentIndex]))
{
builder.Append(':');
builder.Append(lineContent[contentIndex]);
++contentIndex;
}
lastEntry.metadata = builder.ToString();
lastEntry.entity = string.Empty;
lastEntry.topic = string.Empty;
}
else
{
//There is entity but no metadata
if (lineContent[1].StartsWith(" type="))
{
//We check for topic after the entity
lastEntry.topic = string.Empty;
contentIndex = 1;
}
else
{
lastEntry.topic = lineContent[1];
}
lastEntry.entity = lineContent[0];
lastEntry.metadata = string.Empty;
}
builder.Clear();
//Remainder of the log is the message
if (contentIndex < lineContent.Length)
{
builder.Append(lineContent[contentIndex]);
++contentIndex;
}
while (contentIndex < lineContent.Length)
{
builder.Append(':');
builder.Append(lineContent[contentIndex]);
++contentIndex;
}
lastEntry.message = builder.ToString();
}
parsedTime = DateTime.FromFileTimeUtc(parsedTime.ToFileTimeUtc()); // Need to explicitly say log time is in UTC, otherwise it will be interpreted as local
var timeStamp = Timestamp.FromNanoseconds(parsedTime.Ticks * 100);
if (timeStamp < oldestTimestamp)
{
oldestTimestamp = timeStamp;
fileStartTime = parsedTime;
startNanoSeconds = oldestTimestamp.ToNanoseconds;
}
if (timeStamp > newestTImestamp)
{
newestTImestamp = timeStamp;
}
lastEntry.timestamp = new Timestamp(timeStamp.ToNanoseconds - startNanoSeconds);
entriesList.Add(lastEntry);
}
else if (entriesList.Count > 0)
{
if (lastEntry == null)
{
throw new InvalidOperationException("Can't parse the log.");
}
lastEntry.message = lastEntry.message + "\n" + line;
lastEntry.rawLog = lastEntry.rawLog + "\n" + line;
}
++currentLineNumber;
}
if (lastEntry != null)
{
dataProcessor.ProcessDataElement(lastEntry, Context, cancellationToken);
}
contentDictionary[path] = entriesList.AsReadOnly();
file.Close();
--currentLineNumber;
Context.UpdateFileMetadata(path, new FileMetadata(currentLineNumber));
}
var offsetEndTimestamp = new Timestamp(newestTImestamp.ToNanoseconds - startNanoSeconds);
dataSourceInfo = new DataSourceInfo(0, offsetEndTimestamp.ToNanoseconds, fileStartTime);
}
public override void ProcessSource(
ISourceDataProcessor <WaLinuxAgentLogParsedEntry, LogContext, LogParsedDataKey> dataProcessor,
ILogger logger, IProgress <int> progress, CancellationToken cancellationToken)
{
if (FilePaths.Length == 0)
{
return;
}
Timestamp startTime = Timestamp.MaxValue;
Timestamp endTime = Timestamp.MinValue;
DateTime fileStartTime = default(DateTime);
long startNanoSeconds = 0;
foreach (var path in FilePaths)
{
ulong lineNumber = 0;
string line;
var file = new StreamReader(path);
var logEntries = new List <LogEntry>();
LogEntry lastLogEntry = null;
while ((line = file.ReadLine()) != null)
{
lineNumber++;
var WaLinuxAgentMatch = WaLinuxAgentRegex.Match(line);
if (!WaLinuxAgentMatch.Success)
{
// Continuation of the last line.
if (lastLogEntry == null)
{
throw new InvalidOperationException("Can't find the timestamp of the log.");
}
lastLogEntry.Log += '\n' + line;
continue;
}
// We successfully match the format of the log, which means the last log entry is completed.
if (lastLogEntry != null)
{
dataProcessor.ProcessDataElement(lastLogEntry, Context, cancellationToken);
}
DateTime time;
if (!DateTime.TryParse(WaLinuxAgentMatch.Groups[1].Value, out time))
{
throw new InvalidOperationException("Time cannot be parsed to DateTime format");
}
var utcTime = DateTime.FromFileTimeUtc(time.ToFileTimeUtc()); // Need to explicitly say log time is in UTC, otherwise it will be interpreted as local
var timeStamp = Timestamp.FromNanoseconds(utcTime.Ticks * 100);
if (timeStamp < startTime)
{
startTime = timeStamp;
fileStartTime = utcTime;
startNanoSeconds = startTime.ToNanoseconds;
}
if (timeStamp > endTime)
{
endTime = timeStamp;
}
var offsetEventTimestamp = new Timestamp(timeStamp.ToNanoseconds - startNanoSeconds);
lastLogEntry = new LogEntry(path, lineNumber, offsetEventTimestamp,
WaLinuxAgentMatch.Groups[2].Value, WaLinuxAgentMatch.Groups[3].Value);
}
if (lastLogEntry != null)
{
dataProcessor.ProcessDataElement(lastLogEntry, Context, cancellationToken);
}
Context.UpdateFileMetadata(path, new FileMetadata(lineNumber));
}
var offsetEndTimestamp = new Timestamp(endTime.ToNanoseconds - startNanoSeconds);
this.dataSourceInfo = new DataSourceInfo(0, offsetEndTimestamp.ToNanoseconds, fileStartTime);
}