public void ReadFileInOrder(long offSet, LogParserParameters parameters, Dictionary <string, LogMetrics> logMetricsList, LogResponse response) { string fileName = response.LogFile; double timeSpanInMinutes = parameters.EndTime.Subtract(parameters.StartTime).TotalMinutes; bool includeLogs = (timeSpanInMinutes <= 10 && timeSpanInMinutes > 0); long i = 0; using (Stream stream = File.Open(fileName, FileMode.Open)) { stream.Seek(offSet, SeekOrigin.Begin); using (StreamReader reader = new StreamReader(stream)) { string prevLine = ""; bool first = true; var prevDate = new DateTime(); var logDate = new DateTime(); while (!reader.EndOfStream) { string line = reader.ReadLine(); i += line.Length; logDate = GetDateFromLog(line, parameters); if (logDate != new DateTime()) { if (!string.IsNullOrEmpty(prevLine)) { AddLogsToResponse(prevDate, prevLine, includeLogs, parameters, logMetricsList, response); prevLine = line; } if (first) { prevLine = line; first = false; } prevDate = logDate; } else { if (!first) { prevLine = prevLine + Environment.NewLine + line; } ; } if ((logDate != new DateTime() && logDate > parameters.EndTime)) { break; } } //reached the end of the file try last line AddLogsToResponse(prevDate, prevLine, includeLogs, parameters, logMetricsList, response); } } response.DataScanned += i; }
public override DateTime GetDateFromLog(string line, LogParserParameters parameters) { var dateBracket = line.IndexOf("("); if (!line.Contains("GMT") || dateBracket == -1) { return(new DateTime()); } var dateString = string.Join(" ", line.Substring(1, dateBracket - 1).Split().ToList().GetRange(1, 4)); DateTime apiDate = DateTime.Parse(dateString + "Z"); TimeZoneInfo tzi = Util.OlsonTimeZoneToTimeZoneInfo("UTC"); return(TimeZoneInfo.ConvertTime(apiDate, tzi)); }
public override void GetMetricFromLine(DateTime date, string line, LogParserParameters parameters, Dictionary <string, LogMetrics> logMetricsList) { var category = GetCategoryFromLog(line); var metric = new LogMetric(date, 1); if (logMetricsList.ContainsKey(category)) { logMetricsList[category].Add(metric); } else { var newMetricSet = new LogMetrics(); newMetricSet.Add(metric); logMetricsList.Add(category, newMetricSet); } }
public long ReadFileInReverseOrder(string fileName, long offSet, LogParserParameters parameters, LogResponse response) { ReverseLineReader reader = new ReverseLineReader(fileName, offSet); long i = 0; long lastOffset = 0; var lastLine = ""; foreach (var line in reader) { DateTime date = GetDateFromLog(line, parameters); i += line.Length; lastLine += line; if ((date != new DateTime() && date <= parameters.StartTime)) { return(reader.OffSet - line.Length); } } response.DataScanned += i; return(0); }
public override DateTime GetDateFromLog(string line, LogParserParameters parameters) { Util.WriteLog("GetDateFromLog(string line, LogParserParameters parameters)"); var dateBracket = line.IndexOf("]"); if (!line.StartsWith("[") || dateBracket == -1) { Util.WriteLog("returning: " + (new DateTime())); return(new DateTime()); } string dateString = string.Join(" ", line.Substring(1, dateBracket - 1).Split().Take(2)); Util.WriteLog("dateString: " + dateString); string timeZone = line.Substring(1, dateBracket - 1).Split()[2]; Util.WriteLog("timeZone: " + timeZone); DateTime apiDate = DateTime.Parse(dateString + ((timeZone == "UTC") ? "Z" : "")); if (parameters.WorkerType == WorkerType.Linux) { return(Util.ConvertToUtcForLinux(apiDate, timeZone)); } else { if (TimeZoneInfo == null) { TimeZoneInfo = Util.OlsonTimeZoneToTimeZoneInfo(timeZone); } Util.WriteLog("TimeZone " + TimeZoneInfo.DisplayName); if (timeZone == "UTC") { return(TimeZoneInfo.ConvertTime(apiDate, TimeZoneInfo)); } return(TimeZoneInfo.ConvertTimeToUtc(apiDate, TimeZoneInfo)); } }
private void AddLogsToResponse(DateTime prevDate, string prevLine, bool includeLogs, LogParserParameters parameters, Dictionary <string, LogMetrics> logMetricsList, LogResponse response) { var dateLessExceptionStr = RemoveDateFromLog(prevLine).Trim(); if (prevDate >= parameters.StartTime && prevDate <= parameters.EndTime) { GetMetricFromLine(prevDate, dateLessExceptionStr, parameters, logMetricsList); if (!response.ExceptionCount.ContainsKey(dateLessExceptionStr)) { response.ExceptionCount.Add(dateLessExceptionStr, 0); } response.ExceptionCount[dateLessExceptionStr] += 1; if (includeLogs) { response.LinkedLogs.AddLast(prevLine); } } }
public override DateTime GetDateFromLog(string line, LogParserParameters parameters) { var dateBracket = line.IndexOf("]"); if (!line.StartsWith("[") || dateBracket == -1) { return(new DateTime()); } string dateString = string.Join(" ", line.Substring(1, dateBracket - 1).Split().Take(2)); string timeZone = line.Substring(1, dateBracket - 1).Split()[2]; DateTime apiDate = DateTime.Parse(dateString + ((timeZone == "UTC") ? "Z" : "")); TimeZoneInfo tzi = Util.OlsonTimeZoneToTimeZoneInfo(timeZone); if (timeZone == "UTC") { return(TimeZoneInfo.ConvertTime(apiDate, tzi)); } return(TimeZoneInfo.ConvertTimeToUtc(apiDate, tzi)); }
public override void GetMetricFromLine(DateTime date, string line, LogParserParameters parameters, Dictionary <string, LogMetrics> logMetricsList) { throw new NotImplementedException(); }
public override DateTime GetDateFromLog(string line, LogParserParameters parameters) { throw new NotImplementedException(); }
public abstract DateTime GetDateFromLog(string line, LogParserParameters parameters);
public abstract void GetMetricFromLine(DateTime date, string line, LogParserParameters parameters, Dictionary <string, LogMetrics> logMetricsList);
public long BinarySearchLogFile(Stream stream, DateTime dateToQuery, long startingOffset, long endingOffSet, TimeSpan timespan, LogParserParameters parameters) { long middle = (endingOffSet - startingOffset) / 2L; Util.WriteLog("BinarySearchLogFile(Stream stream, DateTime dateToQuery, long startingOffset, long endingOffSet, TimeSpan timespan)"); Util.WriteLog("middle: " + middle); if (middle < 0L) { return(-1L); } stream.Seek(startingOffset + middle, SeekOrigin.Begin); using (StreamReader reader = new StreamReader(stream)) { string readLine; for (string str = ""; !reader.EndOfStream; str = str + readLine) { readLine = reader.ReadLine(); Util.WriteLog("readLine: " + readLine); DateTime dateFromLog = this.GetDateFromLog(readLine, parameters); if (dateFromLog != new DateTime()) { TimeSpan span = dateFromLog.Subtract(dateToQuery); double totalMinutes = Math.Abs(span.TotalMinutes); if ((totalMinutes <= timespan.TotalMinutes)) { return(startingOffset + middle + str.Length); } if (span.TotalMinutes > timespan.TotalMinutes) { long offset = startingOffset + str.Length; return(this.BinarySearchLogFile(stream, dateToQuery, offset, startingOffset + middle, timespan, parameters)); } if ((span.TotalMinutes > 0.0) && (span.TotalMinutes < timespan.TotalMinutes)) { return(startingOffset + str.Length); } if (span.TotalMinutes < 0.0) { long offset = (startingOffset + middle); return(this.BinarySearchLogFile(stream, dateToQuery, offset, endingOffSet, timespan, parameters)); } } } } return(0L); }
public async Task <LogResponse> GetHistogramAsync(LogParserParameters parameters) { Util.WriteLog(DateTime.Now + ": " + "GetHistogramAsync(LogParserParameters parameters)"); Stopwatch sw = new Stopwatch(); sw.Start(); var response = await FindAndSetLoggingFileAndCreateResponseObject(); response.Parameters = parameters; if (!response.LogFileFound) { sw.Stop(); response.ParseTime = sw.ElapsedMilliseconds; return(response); } FileInfo fileInfo = new FileInfo(response.LogFile); long filesize = fileInfo.Length; Util.WriteLog("filesize: " + filesize); long offSet = 0; using (Stream stream = File.Open(response.LogFile, FileMode.Open)) { Util.WriteLog("BinarySearch"); offSet = BinarySearchLogFile(stream, parameters.EndTime, 0, filesize, parameters.EndTime.Subtract(parameters.StartTime), parameters); } Util.WriteLog("offSet: " + offSet); if (offSet == -1) { sw.Stop(); response.ParseTime = sw.ElapsedMilliseconds; return(response); } var logMetricsList = new Dictionary <string, LogMetrics>(); offSet = ReadFileInReverseOrder(response.LogFile, offSet, parameters, response); ReadFileInOrder(offSet, parameters, logMetricsList, response); foreach (var category in logMetricsList.Keys) { var logCatgeorMetrics = new LogMetrics(); logCatgeorMetrics.AddRange(from dt in logMetricsList[category] group dt by dt.Timestamp.Ticks / parameters.TimeGrain.Ticks into g select new LogMetric(Util.GetDateTimeInUtcFormat(new DateTime(g.Key * parameters.TimeGrain.Ticks)), g.ToList().Count)); logCatgeorMetrics.Sort((x, y) => x.Timestamp.CompareTo(y.Timestamp)); response.LogMetrics.Add(category, logCatgeorMetrics); } sw.Stop(); response.ExceptionCount = response.ExceptionCount.OrderByDescending(x => x.Value).ToDictionary(x => x.Key, x => x.Value); response.ParseTime = sw.ElapsedMilliseconds; return(response); }