async Task WriteBufferedTasks(DirSnap dirSnap, CancellationToken cancellationToken) { // todo: pre-allocate array of fixed size and use it to store dequeued write tasks var writeTasks = new List <WriteTask>(_buffer.Count); while (_buffer.TryDequeue(out var writeTask)) { if (writeTask.IsCancelled) { continue; } writeTasks.Add(writeTask); } if (writeTasks.Count == 0) { await Task.Delay(37, cancellationToken); return; } try { await WriteTasksAsync(writeTasks, dirSnap); _logger.Verbose($"Successfully wrote batch of {writeTasks.Count} messages"); writeTasks.ForEach(task => task.Complete()); } catch (Exception exception) { writeTasks.ForEach(task => task.Fail(exception)); } }
async Task RunWorker() { var cancellationToken = _cancellationTokenSource.Token; _logger.Information("Starting writer worker loop"); var dirSnap = new DirSnap(_directoryPath); try { while (!cancellationToken.IsCancellationRequested) { try { await WriteBufferedTasks(dirSnap, cancellationToken); } catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested) { _logger.Verbose("Cancellation detected"); break; } catch (Exception exception) { _logger.Warning(exception, "Exception in worker loop"); } } _logger.Verbose("Exited inner worker loop"); if (_buffer.Count > 0) { _logger.Verbose("Emptying write task buffer"); await WriteBufferedTasks(dirSnap, cancellationToken); } _logger.Verbose("Write task buffer empty"); } catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested) { // ok } catch (Exception exception) { _logger.Error(exception, "Unhandled exception, worker loop failed"); } finally { _logger.Verbose("Cleaning up"); _currentWriter?.Dispose(); _currentWriter = null; _logger.Information("Writer worker loop stopped"); } }
async Task WriteTasksAsync(IEnumerable <WriteTask> writeTasks, DirSnap dirSnap) { if (_currentWriter == null) { string filePath; if (dirSnap.IsEmpty) { filePath = dirSnap.GetFilePath(0); dirSnap.RegisterFile(filePath); } else { filePath = dirSnap.LastFile().FilePath; } var fileInfo = new FileInfo(filePath); _currentWriter = GetWriter(filePath); _approxBytesWritten = fileInfo.Exists ? fileInfo.Length : 0; } var flushNeeded = false; foreach (var task in writeTasks) { foreach (var data in task.Data) { var line = Convert.ToBase64String(data); _currentWriter.Write(line); _currentWriter.WriteLine(LineTerminator); _approxBytesWritten += line.Length + 1; flushNeeded = true; if (_approxBytesWritten > _settings.ApproximateMaximumFileLength) { await _currentWriter.FlushAsync(); flushNeeded = false; _currentWriter.Dispose(); var nextFileNumber = dirSnap.LastFile().FileNumber + 1; var nextFilePath = dirSnap.GetFilePath(nextFileNumber); dirSnap.RegisterFile(nextFilePath); _approxBytesWritten = 0; _currentWriter = GetWriter(nextFilePath); var allFiles = dirSnap.GetFiles().ToList(); var filesToDelete = allFiles .Take(allFiles.Count - _settings.NumberOfFilesToKeep) .ToList(); foreach (var file in filesToDelete) { try { File.Delete(file.FilePath); dirSnap.RemoveFile(file); _logger.Verbose($"Deleted file {file.FilePath}"); } catch (Exception exception) { _logger.Information($"Could not delete file {file.FilePath}: {exception.Message}"); break; } } } } } if (flushNeeded) { await _currentWriter.FlushAsync(); } }