public async Task WhenUnableToCheckpointWithExceptionThenThrows() { // Arrange var context = MockPartitionContext.Create("0", () => { throw new InvalidOperationException(); }); _writerMock .Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>())) .Returns(() => TaskHelpers.CreateCompletedTask(true)); await _processor.OpenAsync(context); // Act await AssertExt.ThrowsAsync <InvalidOperationException>(() => _processor.ProcessEventsAsync(context, new[] { CreateEventData((byte)'a', 100), CreateEventData((byte)'b', 200), CreateEventData((byte)'c', 300), CreateEventData((byte)'d', 400), })); // Assert _loggerMock.Verify(l => l.Warning(It.IsAny <Exception>(), It.IsAny <string>(), It.IsAny <object[]>()), Times.Never); }
public async Task WhenUnableToCheckpointWithStorageExceptionThenLogs() { // Arrange var context = MockPartitionContext.Create("0", () => { throw new StorageException(); }); var processor = new ColdStorageProcessor( n => _writerMock.Object, Mock.Of <IColdStorageInstrumentationPublisher>(), CancellationToken.None, CircuitBreakerWarningLevel, CircuitBreakerStallLevel, CircuitBreakerStallInterval, TimeSpan.FromSeconds(200), "Test", maxBlockSize: MaxBlockSize); _writerMock .Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>())) .Returns(() => TaskHelpers.CreateCompletedTask(true)); await processor.OpenAsync(context); // Act await processor.ProcessEventsAsync(context, new[] { CreateEventData((byte)'a', 100), CreateEventData((byte)'b', 200), CreateEventData((byte)'c', 300), CreateEventData((byte)'d', 400), }); }
public async Task WhenCachedBlocksHitTripLevel_ThenStallsUntilWriteOperationSucceeds() { var blocks = new List <IList <BlockData> >(); Action <IReadOnlyCollection <BlockData>, CancellationToken> saveBlocks = (d, ct) => blocks.Add(d.ToList()); var failWrite = true; _writerMock .Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>())) .Returns(() => TaskHelpers.CreateCompletedTask(!Volatile.Read(ref failWrite))) .Callback(saveBlocks); var context = MockPartitionContext.CreateWithNoopCheckpoint("0"); await _processor.OpenAsync(context); // first N requests fail but go through (the first batch will not fill a frame so it won't result in a write operation) for (int i = 0; i < CircuitBreakerStallLevel + 1; i++) { var batch = new[] { CreateEventData((byte)('a' + i), MaxBlockSize - 200 - i), }; Task processTask = _processor.ProcessEventsAsync(context, batch); await AssertExt.CompletesBeforeTimeoutAsync(processTask, TimeoutInterval); } // N+1th stalls and waits for cached frames to be flushed { var batch = new[] { CreateEventData((byte)('a' + CircuitBreakerStallLevel + 2), 10), }; Task processTask = _processor.ProcessEventsAsync(context, batch); await AssertExt.DoesNotCompleteBeforeTimeoutAsync(processTask, TimeoutInterval); // let the write operation through Volatile.Write(ref failWrite, false); await AssertExt.CompletesBeforeTimeoutAsync(processTask, TimeoutInterval); // check the stalled entries were written var bufferedBlocks = blocks.Last(); Assert.Equal(CircuitBreakerStallLevel, bufferedBlocks.Count); for (int i = 0; i < CircuitBreakerStallLevel; i++) { var lines = GetPayloadsFromBlock(bufferedBlocks[i]); Assert.Equal(1, lines.Length); Assert.Equal( new string((char)('a' + i), MaxBlockSize - 200 - i), lines[0]); } } }
public async Task WhenCachedBlocksRemainBelowTripLevel_ThenDoesNotStall() { var blocks = new List <IList <BlockData> >(); Action <IReadOnlyCollection <BlockData>, CancellationToken> saveBlocks = (d, ct) => blocks.Add(d.ToList()); _writerMock .Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>())) .Returns(() => TaskHelpers.CreateCompletedTask(false)) .Callback(saveBlocks); var context = MockPartitionContext.CreateWithNoopCheckpoint("0"); await _processor.OpenAsync(context); for (int i = 0; i < CircuitBreakerStallLevel; i++) { var batch = new[] { CreateEventData((byte)('a' + i), MaxBlockSize - 200 - i), }; Task processTask = _processor.ProcessEventsAsync(context, batch); await AssertExt.CompletesBeforeTimeoutAsync(processTask, TimeoutInterval); } }
public async Task WhenWritingFullBlockFails_ThenDoesNotThrow() { // Arrange var operationQueue = new List <string>(); Func <Task> flagCheckpointOperation = () => { operationQueue.Add(checkpoint); return(Task.FromResult(true)); }; IList <BlockData> blocks = null; Action <IReadOnlyCollection <BlockData>, CancellationToken> getBlocks = (d, ct) => blocks = d.ToList(); var context = MockPartitionContext.Create("0", flagCheckpointOperation); await _processor.OpenAsync(context); _writerMock .Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>())) .Returns(() => TaskHelpers.CreateCompletedTask(false)) .Callback(getBlocks); // Act await _processor.ProcessEventsAsync( context, new[] { CreateEventData((byte)'a', 100), CreateEventData((byte)'b', 200), CreateEventData((byte)'c', 300), CreateEventData((byte)'d', 400), }); // Assert Assert.NotNull(blocks); Assert.Equal(1, blocks.Count); Assert.False(operationQueue.Contains(checkpoint)); }
public async Task WhenWritingFullBlockFails_ThenReattemptsOnNextFilledBlock() { // Arrange var operationQueue = new List <string>(); Func <Task> flagCheckpointOperation = () => { operationQueue.Add(checkpoint); return(Task.FromResult(true)); }; IList <BlockData> blocks = null; Action <IReadOnlyCollection <BlockData>, CancellationToken> getBlocks = (d, ct) => blocks = d.ToList(); var context = MockPartitionContext.Create("0", flagCheckpointOperation); await _processor.OpenAsync(context); _writerMock .Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>())) .Returns(() => TaskHelpers.CreateCompletedTask(false)) .Callback(getBlocks); // Act await _processor.ProcessEventsAsync( context, new[] { CreateEventData((byte)'a', 100), CreateEventData((byte)'b', 200), CreateEventData((byte)'c', 300), CreateEventData((byte)'d', 400), }); Assert.NotNull(blocks); Assert.Equal(1, blocks.Count); Assert.False(operationQueue.Contains(checkpoint)); await _processor.ProcessEventsAsync( context, new[] { CreateEventData((byte)'e', 300), CreateEventData((byte)'f', 500), }); Assert.NotNull(blocks); Assert.Equal(2, blocks.Count); var serializedFrame = Encoding.UTF8.GetString(blocks[0].Frame, 0, blocks[0].FrameLength); var lines = serializedFrame.Split(new[] { ColdStorageProcessor.EventDelimiter }, StringSplitOptions.RemoveEmptyEntries); Assert.Equal(3, lines.Length); Assert.Equal(new string('a', 100), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[0])).Payload); Assert.Equal(new string('b', 200), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[1])).Payload); Assert.Equal(new string('c', 300), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[2])).Payload); serializedFrame = Encoding.UTF8.GetString(blocks[1].Frame, 0, blocks[1].FrameLength); lines = serializedFrame.Split(new[] { ColdStorageProcessor.EventDelimiter }, StringSplitOptions.RemoveEmptyEntries); Assert.Equal(2, lines.Length); Assert.Equal(new string('d', 400), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[0])).Payload); Assert.Equal(new string('e', 300), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[1])).Payload); }