public void ParseAndSave(ServerLogFileContainer container) { if (!container.AppLogFiles.Any()) return; foreach (AppLogFiles appLogFile in container.AppLogFiles) { DateTime lastEntryDate = DateTime.MinValue; LatestLogFileInfo latestLogFileInfo = null; if (_useSmartParsing) { latestLogFileInfo = _repository.GetLatestLogFileInfo(container.Environment.Name, container.Server.Name, appLogFile.Appname); } if (latestLogFileInfo != null) { lastEntryDate = latestLogFileInfo.DateTime; LogLine("- Using smart update. Last entry for {0}/{1}/{2} was {3}", container.Environment.Name, container.Server.Name, appLogFile.Appname, lastEntryDate); } else { LogLine("- No latest date found for {0} or smart update is off.", appLogFile.Appname); } _appFileDates = new ConcurrentBag<DateTime>(); Parallel.ForEach(appLogFile.LogfilePaths, (filePath) => { LogLine("Parsing {0} ({1})", filePath, ByteSize.FromBytes(new FileInfo(filePath).Length).ToString()); // Smart update - ignore files that are older than the most recent log file FileInfo info = new FileInfo(filePath); _appFileDates.Add(info.LastWriteTimeUtc); if (_useSmartParsing) { if (info.LastWriteTimeUtc > lastEntryDate) { ParseAndSaveSingleLogFile(container.Environment.Name, container.Server.Name, appLogFile.Appname, filePath, latestLogFileInfo); } else { LogLine("Ignoring {0} as it's older than {1} (the last log entry)", filePath, lastEntryDate); } } else { ParseAndSaveSingleLogFile(container.Environment.Name, container.Server.Name, appLogFile.Appname, filePath, null); } }); // Saved the newest log file date LatestLogFileInfo cachedFileInfo = new LatestLogFileInfo() { Id = LatestLogFileInfo.GenerateId(container.Environment.Name, container.Server.Name, appLogFile.Appname), DateTime = _appFileDates.OrderByDescending(x => x.ToUniversalTime()).FirstOrDefault() }; _repository.SaveLatestLogFileInfo(cachedFileInfo); } }
public void SaveLatestLogFileInfo(LatestLogFileInfo latestLogFileInfo) { var collection = _database.GetCollection<LatestLogFileInfo>("LatestLogFileInfo"); collection.ReplaceOneAsync(info => info.Id.Equals(latestLogFileInfo.Id), latestLogFileInfo, new UpdateOptions() { IsUpsert = true}); }
private void ParseAndSaveSingleLogFile(string environment, string server, string appName, string filePath, LatestLogFileInfo latestLogFileInfo) { var list = new List<LogEntry>(); int smartSkipCount = 0; var stringBuilder = new StringBuilder(); using (var streamReader = new StreamReader(new FileStream(filePath, FileMode.Open))) { // Read lines of text in, until we find an "|ERROR" as the delimiter and parse everything up to that line. int lineCount = 0; LogEntry logEntry = null; while (!streamReader.EndOfStream) { string currentLine = streamReader.ReadLine(); if (!string.IsNullOrEmpty(currentLine) && currentLine.Contains("|ERROR") && lineCount > 0) { logEntry = ParseLogEntry(environment, server, appName, stringBuilder.ToString()); if (logEntry != null) { if (_useSmartParsing && latestLogFileInfo != null) { // Check #2 - ignore entries that older than the latest log file date if (logEntry.DateTime > latestLogFileInfo.DateTime) { list.Add(logEntry); } } else { list.Add(logEntry); } } lineCount = 0; stringBuilder = new StringBuilder(); if (list.Count >= MaxEntriesBeforeSave) { LogLine("- Saving {0} items from {1}{2}", list.Count, filePath, _useSmartParsing ? " (smart update skipped " + smartSkipCount+ " entries)" : ""); _repository.BulkSave(list); list = new List<LogEntry>(); GC.Collect(); } } stringBuilder.AppendLine(currentLine); lineCount++; } // Any remaining if (list.Count > 0) { System.Console.WriteLine("- Saving {0} items from {1}", list.Count, filePath); _repository.BulkSave(list); } } }
public void SaveLatestLogFileInfo(LatestLogFileInfo latestLogFileInfo) { }