public void ParseAndSave(ServerLogFileContainer container)
{
if (!container.AppLogFiles.Any())
return;
foreach (AppLogFiles appLogFile in container.AppLogFiles)
{
DateTime lastEntryDate = DateTime.MinValue;
LatestLogFileInfo latestLogFileInfo = null;
if (_useSmartParsing)
{
latestLogFileInfo = _repository.GetLatestLogFileInfo(container.Environment.Name, container.Server.Name, appLogFile.Appname);
}
if (latestLogFileInfo != null)
{
lastEntryDate = latestLogFileInfo.DateTime;
LogLine("- Using smart update. Last entry for {0}/{1}/{2} was {3}", container.Environment.Name, container.Server.Name, appLogFile.Appname, lastEntryDate);
}
else
{
LogLine("- No latest date found for {0} or smart update is off.", appLogFile.Appname);
}
_appFileDates = new ConcurrentBag<DateTime>();
Parallel.ForEach(appLogFile.LogfilePaths, (filePath) =>
{
LogLine("Parsing {0} ({1})", filePath, ByteSize.FromBytes(new FileInfo(filePath).Length).ToString());
// Smart update - ignore files that are older than the most recent log file
FileInfo info = new FileInfo(filePath);
_appFileDates.Add(info.LastWriteTimeUtc);
if (_useSmartParsing)
{
if (info.LastWriteTimeUtc > lastEntryDate)
{
ParseAndSaveSingleLogFile(container.Environment.Name, container.Server.Name, appLogFile.Appname, filePath, latestLogFileInfo);
}
else
{
LogLine("Ignoring {0} as it's older than {1} (the last log entry)", filePath, lastEntryDate);
}
}
else
{
ParseAndSaveSingleLogFile(container.Environment.Name, container.Server.Name, appLogFile.Appname, filePath, null);
}
});
// Saved the newest log file date
LatestLogFileInfo cachedFileInfo = new LatestLogFileInfo()
{
Id = LatestLogFileInfo.GenerateId(container.Environment.Name, container.Server.Name, appLogFile.Appname),
DateTime = _appFileDates.OrderByDescending(x => x.ToUniversalTime()).FirstOrDefault()
};
_repository.SaveLatestLogFileInfo(cachedFileInfo);
}
}
public void SaveLatestLogFileInfo(LatestLogFileInfo latestLogFileInfo)
{
var collection = _database.GetCollection<LatestLogFileInfo>("LatestLogFileInfo");
collection.ReplaceOneAsync(info => info.Id.Equals(latestLogFileInfo.Id), latestLogFileInfo, new UpdateOptions() { IsUpsert = true});
}
private void ParseAndSaveSingleLogFile(string environment, string server, string appName, string filePath, LatestLogFileInfo latestLogFileInfo)
{
var list = new List<LogEntry>();
int smartSkipCount = 0;
var stringBuilder = new StringBuilder();
using (var streamReader = new StreamReader(new FileStream(filePath, FileMode.Open)))
{
// Read lines of text in, until we find an "|ERROR" as the delimiter and parse everything up to that line.
int lineCount = 0;
LogEntry logEntry = null;
while (!streamReader.EndOfStream)
{
string currentLine = streamReader.ReadLine();
if (!string.IsNullOrEmpty(currentLine) && currentLine.Contains("|ERROR") && lineCount > 0)
{
logEntry = ParseLogEntry(environment, server, appName, stringBuilder.ToString());
if (logEntry != null)
{
if (_useSmartParsing && latestLogFileInfo != null)
{
// Check #2 - ignore entries that older than the latest log file date
if (logEntry.DateTime > latestLogFileInfo.DateTime)
{
list.Add(logEntry);
}
}
else
{
list.Add(logEntry);
}
}
lineCount = 0;
stringBuilder = new StringBuilder();
if (list.Count >= MaxEntriesBeforeSave)
{
LogLine("- Saving {0} items from {1}{2}",
list.Count,
filePath,
_useSmartParsing ? " (smart update skipped " + smartSkipCount+ " entries)" : "");
_repository.BulkSave(list);
list = new List<LogEntry>();
GC.Collect();
}
}
stringBuilder.AppendLine(currentLine);
lineCount++;
}
// Any remaining
if (list.Count > 0)
{
System.Console.WriteLine("- Saving {0} items from {1}", list.Count, filePath);
_repository.BulkSave(list);
}
}
}
public void SaveLatestLogFileInfo(LatestLogFileInfo latestLogFileInfo)
{
}