public void CannotCreateMoreThanOneWriter()
{
var logDirectoryPath = GetLogDirectoryPath();
var log = new LogDirectory(logDirectoryPath, new Settings(writeLockAcquisitionTimeoutSeconds: 3));
var writer = log.GetWriter();
Using(writer);
var exception = Assert.Throws <TimeoutException>(() => log.GetWriter());
Console.WriteLine(exception);
}
public async Task CompareSingleVersusMany(bool single, int count)
{
SetLogLevel(LogEventLevel.Information);
var messages = Enumerable.Range(0, count).Select(n => $"THIS IS A STRING MESSAGE EVENT/{n}").ToList();
var logDirectory = new LogDirectory(GetLogDirectoryPath());
var stopwatch = Stopwatch.StartNew();
using (var writer = logDirectory.GetWriter())
{
Log.Information("Writing");
if (single)
{
foreach (var message in messages)
{
await writer.WriteAsync(Encoding.UTF8.GetBytes(message));
}
}
else
{
await writer.WriteManyAsync(messages.Select(Encoding.UTF8.GetBytes));
}
Log.Information("Done writing!");
await Task.Delay(TimeSpan.FromSeconds(0.1));
}
var elapsedSeconds = stopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"Wrote {count} msgs in {elapsedSeconds:0.0} s - that's {count / elapsedSeconds:0.0} msg/s");
}
public async Task CheckResumption(int count)
{
var directoryPath = GetLogDirectoryPath();
var logDirectory = new LogDirectory(directoryPath);
using (var writer = logDirectory.GetWriter())
{
var data = Enumerable.Range(0, count)
.Select(n => $"THIS IS LINE NUMBER {n}")
.Select(Encoding.UTF8.GetBytes);
await writer.WriteManyAsync(data);
}
new DirectoryInfo(directoryPath).DumpDirectoryContentsToConsole();
// read events in a very inefficient way, checking that we can resume at each single line
var fileNumber = -1;
var bytePosition = -1;
for (var counter = 0; counter < count; counter++)
{
var expectedText = $"THIS IS LINE NUMBER {counter}";
var eventData = logDirectory.GetReader().Read(fileNumber, bytePosition).FirstOrDefault();
var actualText = Encoding.UTF8.GetString(eventData.Data);
Assert.That(actualText, Is.EqualTo(expectedText));
fileNumber = eventData.FileNumber;
bytePosition = eventData.BytePosition;
}
}
public async Task CanCreateFilesOfDifferentSize(int approxFileLength)
{
var logDirectoryPath = GetLogDirectoryPath();
var logDirectory = new LogDirectory(logDirectoryPath, new Settings(approximateMaximumFileLength: approxFileLength));
using (var writer = logDirectory.GetWriter())
{
var data = Enumerable.Range(0, 1000)
.Select(n => $"THIS IS LINE NUMBER {n} OUT OF A LOT")
.Select(Encoding.UTF8.GetBytes);
await writer.WriteManyAsync(data);
}
var directory = new DirectoryInfo(logDirectoryPath);
directory.DumpDirectoryContentsToConsole();
var dataFiles = directory.GetFiles("*.dat").OrderBy(f => f.FullName);
foreach (var dataFile in dataFiles)
{
Assert.That(dataFile.Length, Is.LessThan(1.1 * approxFileLength));
}
}
public async Task Writer()
{
var logDirectory = new LogDirectory(@"C:\data\kafkaesque");
// hold on to this bad boy until your application shuts down
using (var logWriter = logDirectory.GetWriter())
{
await logWriter.WriteAsync(new byte[] { 1, 2, 3 });
}
}
Lazy <LogWriter> CreateWriter(string topic)
{
var topicDirectoryPath = Path.Combine(_directoryPath, topic);
return(new Lazy <LogWriter>(() =>
{
var logDirectory = new LogDirectory(topicDirectoryPath, new Settings(logger: new KafkaesqueToToposLogger(_logger)));
_logger.Debug("Initializing new Kafkaesque writer with path {directoryPath}", topicDirectoryPath);
return logDirectory.GetWriter();
}));
}
public async Task CheckBehaviorWhenWriterIsSlow()
{
var logDirectoryPath = GetLogDirectoryPath();
var logDirectory = new LogDirectory(logDirectoryPath, new Settings(logger: new SerilogLogger()));
var writer = Using(logDirectory.GetWriter());
var readEvents = new ConcurrentQueue <string>();
ThreadPool.QueueUserWorkItem(_ =>
{
var cancellationToken = CancelOnDisposal();
try
{
var reader = logDirectory.GetReader();
foreach (var evt in reader.Read(cancellationToken: cancellationToken, throwWhenCancelled: true))
{
var text = Encoding.UTF8.GetString(evt.Data);
Console.WriteLine($"Reader loop read text: {text}");
readEvents.Enqueue(text);
}
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
Console.WriteLine("Reader loop exited");
}
});
async Task Write(string text)
{
Console.WriteLine($"Writing text: {text}");
await writer.WriteAsync(Encoding.UTF8.GetBytes(text));
}
await Task.Run(async() =>
{
await Write("HEJ");
await Task.Delay(TimeSpan.FromSeconds(1));
await Write("MED");
await Task.Delay(TimeSpan.FromSeconds(1));
await Write("DIG");
await Task.Delay(TimeSpan.FromSeconds(1));
await Write("MIN");
await Task.Delay(TimeSpan.FromSeconds(1));
await Write("VÆÆÆÆN");
});
await readEvents.WaitFor(q => q.Count == 5, invariantExpression : q => q.Count >= 0 && q.Count <= 5, timeoutSeconds : 50);
}
public async Task CheckResumption_Eof(int count)
{
var directoryPath = GetLogDirectoryPath();
var logDirectory = new LogDirectory(directoryPath, new Settings(approximateMaximumFileLength: 4096, numberOfFilesToKeep: int.MaxValue));
using (var writer = logDirectory.GetWriter())
{
var data = Enumerable.Range(0, count)
.Select(n => $"THIS IS LINE NUMBER {n}")
.Select(Encoding.UTF8.GetBytes);
await writer.WriteManyAsync(data);
}
new DirectoryInfo(directoryPath).DumpDirectoryContentsToConsole();
// read events in a very inefficient way, checking that we can resume at each single line
var fileNumber = -1;
var bytePosition = -1;
var linesRead = 0;
for (var counter = 0; counter < count; counter++)
{
var expectedText = $"THIS IS LINE NUMBER {counter}";
var data = logDirectory.GetReader().ReadEof(fileNumber, bytePosition).FirstOrDefault();
if (data == LogReader.EOF)
{
break;
}
if (!(data is LogEvent eventData))
{
continue;
}
var actualText = Encoding.UTF8.GetString(eventData.Data);
Assert.That(actualText, Is.EqualTo(expectedText));
fileNumber = eventData.FileNumber;
bytePosition = eventData.BytePosition;
linesRead++;
}
Assert.That(linesRead, Is.EqualTo(count));
}
public async Task CanReadSomeEvents()
{
var logDirectoryPath = GetLogDirectoryPath();
var log = new LogDirectory(logDirectoryPath);
var reader = log.GetReader();
using var writer = log.GetWriter();
await writer.WriteAsync(new byte[] { 1, 2, 3 });
await writer.WriteAsync(new byte[] { 1, 2, 3 });
await writer.WriteAsync(new byte[] { 1, 2, 3 });
var list = reader.ReadEof().TakeWhile(e => e != LogReader.EOF).ToList();
Assert.That(list.Count, Is.EqualTo(3));
}
public async Task WhatHappensIfWeWriteALot(int iterations, bool parallel)
{
SetLogLevel(LogEventLevel.Information);
var logDirectoryPath = GetLogDirectoryPath();
var logDirectory = new LogDirectory(logDirectoryPath);
var logWriter = logDirectory.GetWriter();
Using(logWriter);
var bytes = Enumerable.Range(0, 20000)
.Select(o => (byte)(iterations % 256))
.ToArray();
var stopwatch = Stopwatch.StartNew();
if (parallel)
{
await Task.WhenAll(Enumerable.Range(0, iterations)
.Select(i => logWriter.WriteAsync(bytes)));
}
else
{
for (var counter = 0; counter < iterations; counter++)
{
await logWriter.WriteAsync(bytes);
}
}
var directoryInfo = new DirectoryInfo(logDirectoryPath);
directoryInfo.DumpDirectoryContentsToConsole();
var files = directoryInfo.GetFiles();
var elapsedSeconds = stopwatch.Elapsed.TotalSeconds;
var totalBytesWritten = files.Sum(a => a.Length);
Console.WriteLine($"Wrote {totalBytesWritten.FormatAsHumanReadableSize()} in {elapsedSeconds:0.0} s - that's {((long)(totalBytesWritten / elapsedSeconds)).FormatAsHumanReadableSize()}/s");
}
public async Task CanWriteAndReadItBack()
{
var logDirectoryPath = GetLogDirectoryPath();
var logDirectory = new LogDirectory(logDirectoryPath);
var logWriter = logDirectory.GetWriter();
Using(logWriter);
await logWriter.WriteAsync(new byte[] { 1, 2, 3 }, CancelAfter(TimeSpan.FromSeconds(3)));
var reader = logDirectory.GetReader();
var logEvents = reader.Read(cancellationToken: CancelAfter(TimeSpan.FromSeconds(3))).ToList();
Assert.That(logEvents.Count, Is.EqualTo(1));
var logEvent = logEvents.First();
Assert.That(logEvent.Data, Is.EqualTo(new byte[] { 1, 2, 3 }));
}
public async Task CanDeleteOldFiles()
{
var directoryInfo = new DirectoryInfo(GetLogDirectoryPath());
var settings = new Settings(numberOfFilesToKeep: 10, approximateMaximumFileLength: 32768);
var logDirectory = new LogDirectory(directoryInfo, settings);
using (var writer = logDirectory.GetWriter())
{
var data = Enumerable.Range(0, 10000)
.Select(n => $"THIS IS LINE NUMBER {n} OUT OF QUITE A FEW")
.Select(Encoding.UTF8.GetBytes);
await writer.WriteManyAsync(data);
}
directoryInfo.DumpDirectoryContentsToConsole();
var dataFiles = directoryInfo.GetFiles("*.dat").ToList();
Assert.That(dataFiles.Count, Is.EqualTo(10));
}
public async Task CanReadAndResumeAfterExperiencingEof()
{
var logDirectoryPath = GetLogDirectoryPath();
var log = new LogDirectory(logDirectoryPath);
var reader = log.GetReader();
using var writer = log.GetWriter();
await writer.WriteAsync(new byte[] { 1, 2, 3 });
await writer.WriteAsync(new byte[] { 1, 2, 3 });
await writer.WriteAsync(new byte[] { 1, 2, 3 });
var firstList = reader.ReadEof().TakeWhile(e => e != LogReader.EOF).Cast <LogEvent>().ToList();
await writer.WriteAsync(new byte[] { 1, 2, 3 });
await writer.WriteAsync(new byte[] { 1, 2, 3 });
await writer.WriteAsync(new byte[] { 1, 2, 3 });
await writer.WriteAsync(new byte[] { 1, 2, 3 });
await writer.WriteAsync(new byte[] { 1, 2, 3 });
var(fileNumber, bytePosition) = (firstList.Last().FileNumber, firstList.Last().BytePosition);
var secondList = reader.ReadEof(fileNumber, bytePosition)
.TakeWhile(e => e != LogReader.EOF)
.ToList();
Assert.That(firstList.Count, Is.EqualTo(3));
Assert.That(secondList.Count, Is.EqualTo(5));
}
static async Task Main()
{
Log.Logger = new LoggerConfiguration()
.WriteTo.ColoredConsole()
.MinimumLevel.Verbose()
.CreateLogger();
var count = 100;
var logDirectory = new LogDirectory(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "data"));
var stopwatch = Stopwatch.StartNew();
using (var writer = logDirectory.GetWriter())
{
var messages = Enumerable.Range(0, count).Select(n => $"THIS IS MESSAGE NUMBER {n}");
await writer.WriteManyAsync(messages.Select(Encoding.UTF8.GetBytes));
//await Task.Delay(TimeSpan.FromSeconds(.1));
}
var elapsedSeconds = stopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"Wrote {count} messages in {elapsedSeconds:0.0} s - that's {count/elapsedSeconds:0.0} msg/s");
}
async Task RunTest(int count, int readerCount)
{
SetLogLevel(LogEventLevel.Information);
var logDirectoryPath = GetLogDirectoryPath();
var logDirectory = new LogDirectory(logDirectoryPath);
var messagesToWrite = Enumerable.Range(0, count)
.Select(n => $"THIS IS A STRING MESSAGE/{n}")
.ToConcurrentQueue();
var cancellationTokenSource = new CancellationTokenSource();
Using(cancellationTokenSource);
var cancellationToken = cancellationTokenSource.Token;
var writer = logDirectory.GetWriter();
Using(writer);
var readerThreads = Enumerable
.Range(0, readerCount)
.Select(n =>
{
var logReader = logDirectory.GetReader();
var readMessages = new ConcurrentQueue <string>();
return(new
{
Thread = new Thread(() =>
{
try
{
foreach (var logEvent in logReader.Read(cancellationToken: cancellationToken))
{
readMessages.Enqueue(Encoding.UTF8.GetString(logEvent.Data));
}
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
Console.WriteLine($"Thread {n} was cancelled");
}
catch (Exception exception)
{
Console.WriteLine(exception);
}
})
{
IsBackground = true,
Name = $"Reader thread {n}"
},
Messages = readMessages
});
})
.ToList();
readerThreads.ForEach(a => a.Thread.Start());
var writerThreads = Enumerable.Range(0, 10)
.Select(n => new Thread(() =>
{
while (messagesToWrite.TryDequeue(out var message))
{
writer.WriteAsync(Encoding.UTF8.GetBytes(message), cancellationToken);
}
})
{
Name = $"Writer thread {n}"
})
public async Task CanReadBackEventsSpreadOverMultipleFiles_WritingEverythingInAdvance(int count)
{
SetLogLevel(LogEventLevel.Verbose);
var messages = Enumerable.Range(0, count)
.Select(n => $"{n}/This is a pretty long string message, whose purpose is solely to take up a lot of space, meaning that the events will eventually need to be placed in more than one file.");
var logDirectoryPath = GetLogDirectoryPath();
var directoryInfo = new DirectoryInfo(logDirectoryPath);
var logDirectory = new LogDirectory(directoryInfo);
var writeStopwatch = Stopwatch.StartNew();
// write everything
var writer = logDirectory.GetWriter();
Using(writer);
await writer.WriteManyAsync(messages.Select(Encoding.UTF8.GetBytes));
var elapsedSecondsWriting = writeStopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"Wrote {count} messages in {elapsedSecondsWriting:0.0} s - that's {count/elapsedSecondsWriting:0.0} msg/s");
directoryInfo.DumpDirectoryContentsToConsole();
// read it back
var reader = logDirectory.GetReader();
var readStopwatch = Stopwatch.StartNew();
var expectedMessageNumber = 0;
foreach (var message in reader.Read(cancellationToken: CancelAfter(TimeSpan.FromSeconds(20))).Take(count))
{
var text = Encoding.UTF8.GetString(message.Data);
var parts = text.Split('/');
try
{
if (parts.Length != 2)
{
throw new FormatException(
$"The text '{text}' could not be parsed - expected a number and a slash, followed by some text");
}
if (!int.TryParse(parts.First(), out var actualMessageNumber))
{
throw new FormatException(
$"Could not parse the token '{parts.First()}' from the message '{text}' into an integer");
}
if (actualMessageNumber != expectedMessageNumber)
{
throw new AssertionException(
$"The message number {actualMessageNumber} did not match the expected: {expectedMessageNumber}");
}
}
catch (Exception exception)
{
throw new ApplicationException($"Error processing event with fileNumber = {message.FileNumber}, bytePosition = {message.BytePosition}", exception);
}
expectedMessageNumber++;
}
var elapsedSeconds = readStopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"Read {count} messages in {elapsedSeconds:0.0} s - that's {count/elapsedSeconds:0.0} msg/s");
Assert.That(expectedMessageNumber, Is.EqualTo(count));
}
public async Task CanReadBackEventsSpreadOverMultipleFiles_ReadingWhileWriting(int count)
{
SetLogLevel(LogEventLevel.Verbose);
var messages = Enumerable.Range(0, count)
.Select(n => $"{n}/This is a pretty long string message, whose purpose is solely to take up a lot of space, meaning that the events will eventually need to be placed in more than one file.");
var logDirectoryPath = GetLogDirectoryPath();
var directoryInfo = new DirectoryInfo(logDirectoryPath);
var logDirectory = new LogDirectory(directoryInfo);
var doneWriting = Using(new ManualResetEvent(false));
var writer = logDirectory.GetWriter();
// ensure that the background thread has finished writing before we dispose the writer
Using(new DisposableCallback(() =>
{
using (writer)
{
var timeout = TimeSpan.FromMinutes(1);
if (!doneWriting.WaitOne(timeout))
{
Console.WriteLine($"WARNING: WRITE OPERATION WAS NOT COMPLETED WITHIN {timeout} TIMEOUT");
}
}
}));
ThreadPool.QueueUserWorkItem(async _ =>
{
try
{
await writer.WriteManyAsync(messages.Select(Encoding.UTF8.GetBytes));
directoryInfo.DumpDirectoryContentsToConsole();
}
finally
{
doneWriting.Set();
}
});
var reader = logDirectory.GetReader();
var expectedMessageNumber = 0;
var stopwatch = Stopwatch.StartNew();
foreach (var message in reader.Read(cancellationToken: CancelAfter(TimeSpan.FromSeconds(20))).Take(count))
{
var text = Encoding.UTF8.GetString(message.Data);
var parts = text.Split('/');
try
{
if (parts.Length != 2)
{
throw new FormatException(
$"The text '{text}' could not be parsed - expected a number and a slash, followed by some text");
}
if (!int.TryParse(parts.First(), out var actualMessageNumber))
{
throw new FormatException(
$"Could not parse the token '{parts.First()}' from the message '{text}' into an integer");
}
if (actualMessageNumber != expectedMessageNumber)
{
throw new AssertionException(
$"The message number {actualMessageNumber} did not match the expected: {expectedMessageNumber}");
}
}
catch (Exception exception)
{
throw new ApplicationException($"Error processing event with fileNumber = {message.FileNumber}, bytePosition = {message.BytePosition}", exception);
}
expectedMessageNumber++;
}
var elapsedSeconds = stopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"Read {count} messages in {elapsedSeconds:0.0} s - that's {count/elapsedSeconds:0.0} msg/s");
Assert.That(expectedMessageNumber, Is.EqualTo(count));
}