public void ShouldDiscardLogs()
{
LogReader reader;
Log log;
StringMatcher stringMatcher;
StringMatcher discardMatcher;
stringMatcher = new StringMatcher();
stringMatcher.Add("Line");
discardMatcher = new StringMatcher();
discardMatcher.Add("Line 1");
discardMatcher.Add("Line 3");
discardMatcher.Add("Line 5");
discardMatcher.Add("Line 7");
discardMatcher.Add("Line 9");
reader = new LogReader(new MockedLineReader(10), stringMatcher, discardMatcher);
for (int t = 0; t < 5; t++)
{
log = reader.Read();
Assert.AreEqual(t * 2, log.LineIndex);
Assert.AreEqual($"Line {t*2}", log.ToSingleLine());
}
log = reader.Read();
Assert.IsNull(log);
}
public void ShouldReadAfterDiscardLogs() // ex: used when a log file starts with comment
{
LogReader reader;
Log log;
StringMatcher stringMatcher;
StringMatcher discardMatcher;
stringMatcher = new StringMatcher();
stringMatcher.Add("Line");
discardMatcher = new StringMatcher();
discardMatcher.Add("Line 0");
discardMatcher.Add("Line 1");
discardMatcher.Add("Line 2");
discardMatcher.Add("Line 3");
reader = new LogReader(new MockedLineReader(10), stringMatcher, discardMatcher);
for (int t = 0; t < 6; t++)
{
log = reader.Read();
Assert.AreEqual(t + 4, log.LineIndex);
Assert.AreEqual($"Line {t+4}", log.ToSingleLine());
}
log = reader.Read();
Assert.IsNull(log);
}
public void ReadTest8()
{
var log = "2010-04-25 09:49:31 GET /index.asp 183.64.134.12 HTTP/1.1 200 220202" + Environment.NewLine
+ "2010-04-25 09:49:32 GET /index.asp 183.64.134.15 HTTP/1.1 200 220202";
var reader = new LogReader(Helper.CreateStreamReader(log));
var line = reader.Read();
Assert.NotNull(line);
Assert.Equal(DateTime.Parse("2010-04-25 09:49:31"), line.Created);
Assert.Equal("183.64.134.12", line.IPAddress);
line = reader.Read();
Assert.NotNull(line);
Assert.Equal(DateTime.Parse("2010-04-25 09:49:32"), line.Created);
Assert.Equal("183.64.134.15", line.IPAddress);
}
public void Reader_Works()
{
var expected = Input1Logs;
var result = LogReader.Read(InputPath1);
CollectionAssert.AreEqual(expected, result);
}
public void Will_Merge_Three_In_Order() // Also contains 2 logs with the same timestamp
{
var expected = LogReader.Read(OutputPath123);
var result = LogMerger.LogMerger.Merge(new string[] { InputPath1, InputPath2, InputPath3 });
CollectionAssert.AreEqual(expected, result);
}
public void Will_Merge_Two_In_Order()
{
var expected = LogReader.Read(OutputPath12);
var result = LogMerger.LogMerger.Merge(new string[] { InputPath1, InputPath2 });
CollectionAssert.AreEqual(expected, result);
}
public void Duplicate_Input()
{
var expected = LogReader.Read(InputPath1);
var result = LogMerger.LogMerger.Merge(new string[] { InputPath1, InputPath1 });
CollectionAssert.AreEqual(expected, result);
}
static async Task Main(string[] args)
{
Console.WriteLine($"Starting up Leaf Log Reader...");
LoadSettings(args);
CheckDirectories();
await CheckDatabase();
var manager = new LogEntryTransferManager(settings);
var reader = new LogReader(settings, manager);
if (!reader.FilesFound)
{
Console.WriteLine($"No files found to parse. Exiting...");
return;
}
while (reader.Read())
{
await reader.Process();
}
Console.WriteLine($"Successfully copied {manager.CopiedCount} log entries from {reader.FileCount} file(s).");
Console.WriteLine($"Exiting...");
}
public void ReadTest1()
{
var log = "";
var reader = new LogReader(Helper.CreateStreamReader(log));
var line = reader.Read();
Assert.Null(line);
}
public void ReadTest5()
{
var log = "2010-04-25 09:49:31 GET /index.asp 183.64.134";
var reader = new LogReader(Helper.CreateStreamReader(log));
var line = reader.Read();
Assert.Null(line);
}
public void Read_CanReadAllLinesInLog()
{
var logReader = new LogReader();
string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"LogMailer\test-web.txt");
var log = logReader.Read(path);
Assert.AreEqual(6, log.Count());
}
// TODO: Send log messages in batches
public async Task Start()
{
await foreach (var message in _reader.Read(_stoppingToken))
{
await _httpClientFactory
.CreateClient("LoggerJob")
.PostAsJsonAsync("/logs", message, cancellationToken: _stoppingToken);
}
}
/**
* Read the text from the log file from default location
*/
public string GetLogText()
{
LogReader logReader = new LogReader();
logReader.OpenReader();
string text = logReader.Read();
logReader.CloseReader();
return(text);
}
/**
* Read the text from the log file from custom location
*/
public string GetLogText(string location)
{
LogReader logReader = new LogReader();
logReader.LogLocation = location;
logReader.OpenReader();
string text = logReader.Read();
logReader.CloseReader();
return(text);
}
public void Writer_Works()
{
var expected = LogReader.Read(InputPath1);
var testPath = "writer-test.txt";
FileLogWriter.Write(testPath, Input1Logs);
var result = LogReader.Read(testPath);
CollectionAssert.AreEqual(expected, result);
}
public void ShouldReadLogs()
{
LogReader reader;
Log log;
StringMatcher stringMatcher;
stringMatcher = new StringMatcher();
stringMatcher.Add("Line");
reader = new LogReader(new MockedLineReader(10), stringMatcher, Utils.EmptyStringMatcher);
for (int t = 0; t < 10; t++)
{
log = reader.Read();
Assert.AreEqual(t, log.LineIndex);
Assert.AreEqual($"Line {t}", log.ToSingleLine());
}
log = reader.Read();
Assert.IsNull(log);
}
public void StartReadLog()
{
totalReadTimeWatch.Restart();
if (this.minFilter > this.maxFilter && this.maxFilter >= LogLevel.Trace)
{
throw new ArgumentException(Resources.MinLevelGreaterThenMax);
}
var reader = new LogReader(this.charsetDetector, this.matcher, this.filter);
var logPath = this.view.LogPath;
var length = new FileInfo(logPath).Length;
var append = length > this.logSize && this.CurrentPathCached;
var offset = append ? this.logSize : 0L;
this.logSize = length;
if (this.logSize == 0)
{
throw new ArgumentException(Resources.ZeroFileDetected);
}
if (this.CurrentPathCached && !append)
{
this.ReadLogFromInternalStore(true);
return;
}
this.currentPath = logPath;
if (this.currentPath == null)
{
return;
}
this.RunOnGuiThread(this.SetLogSize);
this.RunOnGuiThread(this.ResetLogStatistic);
var dbSize = this.logSize + (this.logSize / 10) * 4; // +40% to log file
if (this.store != null && !append)
{
this.store.Dispose();
}
if (!append || this.store == null)
{
this.store = new LogStore(dbSize, null, this.matcher.MessageSchema);
}
GC.Collect();
this.store.StartAddMessages();
if (!append)
{
this.totalMessages = 0;
}
reader.ProgressChanged += this.OnReadLogProgressChanged;
reader.CompilationStarted += OnCompilationStarted;
reader.CompilationFinished += this.OnCompilationFinished;
reader.EncodingDetectionStarted += this.OnEncodingDetectionStarted;
reader.EncodingDetectionFinished += this.OnEncodingDetectionFinished;
Encoding inputEncoding;
this.filesEncodingCache.TryGetValue(this.currentPath, out inputEncoding);
try
{
this.queuedMessages = 0;
var encoding = reader.Read(logPath, this.AddMessageToCache, () => this.NotCancelled, inputEncoding, offset);
this.probeWatch.Stop();
var elapsed = this.probeWatch.Elapsed;
var pending = Interlocked.Read(ref this.queuedMessages);
var inserted = this.totalMessages - pending;
var insertRatio = inserted / elapsed.TotalSeconds;
var remain = Math.Abs(insertRatio) < 0.00001
? TimeSpan.FromSeconds(0)
: TimeSpan.FromSeconds(pending / insertRatio);
if (this.currentPath != null && !this.filesEncodingCache.ContainsKey(this.currentPath) &&
encoding != null)
{
this.filesEncodingCache.Add(this.currentPath, encoding);
}
var remainSeconds = remain.Seconds;
if (remainSeconds > 0)
{
this.RunOnGuiThread(
() => this.view.SetLogProgressCustomText(string.Format(Resources.FinishLoading, remainSeconds)));
}
// Interlocked is a must because other threads can change this
SpinWait.SpinUntil(
() => Interlocked.Read(ref this.queuedMessages) == 0 || this.cancellation.IsCancellationRequested);
}
finally
{
this.RunOnGuiThread(
() => this.view.SetLogProgressCustomText(Resources.LogIndexing));
this.store.FinishAddMessages();
reader.ProgressChanged -= this.OnReadLogProgressChanged;
reader.CompilationStarted -= this.OnCompilationStarted;
reader.CompilationFinished -= this.OnCompilationFinished;
reader.EncodingDetectionStarted -= this.OnEncodingDetectionStarted;
reader.EncodingDetectionFinished -= this.OnEncodingDetectionFinished;
}
this.ReadLogFromInternalStore(false);
}
