async Task WriteBufferedTasks(DirSnap dirSnap, CancellationToken cancellationToken)
{
// todo: pre-allocate array of fixed size and use it to store dequeued write tasks
var writeTasks = new List <WriteTask>(_buffer.Count);
while (_buffer.TryDequeue(out var writeTask))
{
if (writeTask.IsCancelled)
{
continue;
}
writeTasks.Add(writeTask);
}
if (writeTasks.Count == 0)
{
await Task.Delay(37, cancellationToken);
return;
}
try
{
await WriteTasksAsync(writeTasks, dirSnap);
_logger.Verbose($"Successfully wrote batch of {writeTasks.Count} messages");
writeTasks.ForEach(task => task.Complete());
}
catch (Exception exception)
{
writeTasks.ForEach(task => task.Fail(exception));
}
}
async Task RunWorker()
{
var cancellationToken = _cancellationTokenSource.Token;
_logger.Information("Starting writer worker loop");
var dirSnap = new DirSnap(_directoryPath);
try
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
await WriteBufferedTasks(dirSnap, cancellationToken);
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
_logger.Verbose("Cancellation detected");
break;
}
catch (Exception exception)
{
_logger.Warning(exception, "Exception in worker loop");
}
}
_logger.Verbose("Exited inner worker loop");
if (_buffer.Count > 0)
{
_logger.Verbose("Emptying write task buffer");
await WriteBufferedTasks(dirSnap, cancellationToken);
}
_logger.Verbose("Write task buffer empty");
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
// ok
}
catch (Exception exception)
{
_logger.Error(exception, "Unhandled exception, worker loop failed");
}
finally
{
_logger.Verbose("Cleaning up");
_currentWriter?.Dispose();
_currentWriter = null;
_logger.Information("Writer worker loop stopped");
}
}
async Task WriteTasksAsync(IEnumerable <WriteTask> writeTasks, DirSnap dirSnap)
{
if (_currentWriter == null)
{
string filePath;
if (dirSnap.IsEmpty)
{
filePath = dirSnap.GetFilePath(0);
dirSnap.RegisterFile(filePath);
}
else
{
filePath = dirSnap.LastFile().FilePath;
}
var fileInfo = new FileInfo(filePath);
_currentWriter = GetWriter(filePath);
_approxBytesWritten = fileInfo.Exists ? fileInfo.Length : 0;
}
var flushNeeded = false;
foreach (var task in writeTasks)
{
foreach (var data in task.Data)
{
var line = Convert.ToBase64String(data);
_currentWriter.Write(line);
_currentWriter.WriteLine(LineTerminator);
_approxBytesWritten += line.Length + 1;
flushNeeded = true;
if (_approxBytesWritten > _settings.ApproximateMaximumFileLength)
{
await _currentWriter.FlushAsync();
flushNeeded = false;
_currentWriter.Dispose();
var nextFileNumber = dirSnap.LastFile().FileNumber + 1;
var nextFilePath = dirSnap.GetFilePath(nextFileNumber);
dirSnap.RegisterFile(nextFilePath);
_approxBytesWritten = 0;
_currentWriter = GetWriter(nextFilePath);
var allFiles = dirSnap.GetFiles().ToList();
var filesToDelete = allFiles
.Take(allFiles.Count - _settings.NumberOfFilesToKeep)
.ToList();
foreach (var file in filesToDelete)
{
try
{
File.Delete(file.FilePath);
dirSnap.RemoveFile(file);
_logger.Verbose($"Deleted file {file.FilePath}");
}
catch (Exception exception)
{
_logger.Information($"Could not delete file {file.FilePath}: {exception.Message}");
break;
}
}
}
}
}
if (flushNeeded)
{
await _currentWriter.FlushAsync();
}
}
