public async Task WhenUnableToCheckpointWithExceptionThenThrows()
{
// Arrange
var context = MockPartitionContext.Create("0", () =>
{
throw new InvalidOperationException();
});
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(() => TaskHelpers.CreateCompletedTask(true));
await _processor.OpenAsync(context);
// Act
await AssertExt.ThrowsAsync <InvalidOperationException>(() => _processor.ProcessEventsAsync(context, new[]
{
CreateEventData((byte)'a', 100),
CreateEventData((byte)'b', 200),
CreateEventData((byte)'c', 300),
CreateEventData((byte)'d', 400),
}));
// Assert
_loggerMock.Verify(l => l.Warning(It.IsAny <Exception>(), It.IsAny <string>(), It.IsAny <object[]>()), Times.Never);
}
public async Task WhenUnableToCheckpointWithStorageExceptionThenLogs()
{
// Arrange
var context = MockPartitionContext.Create("0", () =>
{
throw new StorageException();
});
var processor =
new ColdStorageProcessor(
n => _writerMock.Object,
Mock.Of <IColdStorageInstrumentationPublisher>(),
CancellationToken.None,
CircuitBreakerWarningLevel,
CircuitBreakerStallLevel,
CircuitBreakerStallInterval,
TimeSpan.FromSeconds(200),
"Test",
maxBlockSize: MaxBlockSize);
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(() => TaskHelpers.CreateCompletedTask(true));
await processor.OpenAsync(context);
// Act
await processor.ProcessEventsAsync(context, new[]
{
CreateEventData((byte)'a', 100),
CreateEventData((byte)'b', 200),
CreateEventData((byte)'c', 300),
CreateEventData((byte)'d', 400),
});
}
public async Task WhenCachedBlocksHitTripLevel_ThenStallsUntilWriteOperationSucceeds()
{
var blocks = new List <IList <BlockData> >();
Action <IReadOnlyCollection <BlockData>, CancellationToken> saveBlocks = (d, ct) => blocks.Add(d.ToList());
var failWrite = true;
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(() => TaskHelpers.CreateCompletedTask(!Volatile.Read(ref failWrite)))
.Callback(saveBlocks);
var context = MockPartitionContext.CreateWithNoopCheckpoint("0");
await _processor.OpenAsync(context);
// first N requests fail but go through (the first batch will not fill a frame so it won't result in a write operation)
for (int i = 0; i < CircuitBreakerStallLevel + 1; i++)
{
var batch = new[] { CreateEventData((byte)('a' + i), MaxBlockSize - 200 - i), };
Task processTask = _processor.ProcessEventsAsync(context, batch);
await AssertExt.CompletesBeforeTimeoutAsync(processTask, TimeoutInterval);
}
// N+1th stalls and waits for cached frames to be flushed
{
var batch = new[] { CreateEventData((byte)('a' + CircuitBreakerStallLevel + 2), 10), };
Task processTask = _processor.ProcessEventsAsync(context, batch);
await AssertExt.DoesNotCompleteBeforeTimeoutAsync(processTask, TimeoutInterval);
// let the write operation through
Volatile.Write(ref failWrite, false);
await AssertExt.CompletesBeforeTimeoutAsync(processTask, TimeoutInterval);
// check the stalled entries were written
var bufferedBlocks = blocks.Last();
Assert.Equal(CircuitBreakerStallLevel, bufferedBlocks.Count);
for (int i = 0; i < CircuitBreakerStallLevel; i++)
{
var lines = GetPayloadsFromBlock(bufferedBlocks[i]);
Assert.Equal(1, lines.Length);
Assert.Equal(
new string((char)('a' + i), MaxBlockSize - 200 - i),
lines[0]);
}
}
}
public async Task WhenCachedBlocksRemainBelowTripLevel_ThenDoesNotStall()
{
var blocks = new List <IList <BlockData> >();
Action <IReadOnlyCollection <BlockData>, CancellationToken> saveBlocks = (d, ct) => blocks.Add(d.ToList());
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(() => TaskHelpers.CreateCompletedTask(false))
.Callback(saveBlocks);
var context = MockPartitionContext.CreateWithNoopCheckpoint("0");
await _processor.OpenAsync(context);
for (int i = 0; i < CircuitBreakerStallLevel; i++)
{
var batch = new[] { CreateEventData((byte)('a' + i), MaxBlockSize - 200 - i), };
Task processTask = _processor.ProcessEventsAsync(context, batch);
await AssertExt.CompletesBeforeTimeoutAsync(processTask, TimeoutInterval);
}
}
public async Task WhenWritingFullBlockFails_ThenDoesNotThrow()
{
// Arrange
var operationQueue = new List <string>();
Func <Task> flagCheckpointOperation = () =>
{
operationQueue.Add(checkpoint);
return(Task.FromResult(true));
};
IList <BlockData> blocks = null;
Action <IReadOnlyCollection <BlockData>, CancellationToken> getBlocks = (d, ct) => blocks = d.ToList();
var context = MockPartitionContext.Create("0", flagCheckpointOperation);
await _processor.OpenAsync(context);
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(() => TaskHelpers.CreateCompletedTask(false))
.Callback(getBlocks);
// Act
await _processor.ProcessEventsAsync(
context,
new[]
{
CreateEventData((byte)'a', 100),
CreateEventData((byte)'b', 200),
CreateEventData((byte)'c', 300),
CreateEventData((byte)'d', 400),
});
// Assert
Assert.NotNull(blocks);
Assert.Equal(1, blocks.Count);
Assert.False(operationQueue.Contains(checkpoint));
}
public async Task WhenWritingFullBlockFails_ThenReattemptsOnNextFilledBlock()
{
// Arrange
var operationQueue = new List <string>();
Func <Task> flagCheckpointOperation = () =>
{
operationQueue.Add(checkpoint);
return(Task.FromResult(true));
};
IList <BlockData> blocks = null;
Action <IReadOnlyCollection <BlockData>, CancellationToken> getBlocks = (d, ct) => blocks = d.ToList();
var context = MockPartitionContext.Create("0", flagCheckpointOperation);
await _processor.OpenAsync(context);
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(() => TaskHelpers.CreateCompletedTask(false))
.Callback(getBlocks);
// Act
await _processor.ProcessEventsAsync(
context,
new[]
{
CreateEventData((byte)'a', 100),
CreateEventData((byte)'b', 200),
CreateEventData((byte)'c', 300),
CreateEventData((byte)'d', 400),
});
Assert.NotNull(blocks);
Assert.Equal(1, blocks.Count);
Assert.False(operationQueue.Contains(checkpoint));
await _processor.ProcessEventsAsync(
context,
new[]
{
CreateEventData((byte)'e', 300),
CreateEventData((byte)'f', 500),
});
Assert.NotNull(blocks);
Assert.Equal(2, blocks.Count);
var serializedFrame = Encoding.UTF8.GetString(blocks[0].Frame, 0, blocks[0].FrameLength);
var lines = serializedFrame.Split(new[] { ColdStorageProcessor.EventDelimiter }, StringSplitOptions.RemoveEmptyEntries);
Assert.Equal(3, lines.Length);
Assert.Equal(new string('a', 100), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[0])).Payload);
Assert.Equal(new string('b', 200), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[1])).Payload);
Assert.Equal(new string('c', 300), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[2])).Payload);
serializedFrame = Encoding.UTF8.GetString(blocks[1].Frame, 0, blocks[1].FrameLength);
lines = serializedFrame.Split(new[] { ColdStorageProcessor.EventDelimiter }, StringSplitOptions.RemoveEmptyEntries);
Assert.Equal(2, lines.Length);
Assert.Equal(new string('d', 400), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[0])).Payload);
Assert.Equal(new string('e', 300), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[1])).Payload);
}
