public async Task WhenUnableToCheckpointWithExceptionThenThrows()
{
// Arrange
var context = MockPartitionContext.Create("0", () =>
{
throw new InvalidOperationException();
});
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(() => TaskHelpers.CreateCompletedTask(true));
await _processor.OpenAsync(context);
// Act
await AssertExt.ThrowsAsync <InvalidOperationException>(() => _processor.ProcessEventsAsync(context, new[]
{
CreateEventData((byte)'a', 100),
CreateEventData((byte)'b', 200),
CreateEventData((byte)'c', 300),
CreateEventData((byte)'d', 400),
}));
// Assert
_loggerMock.Verify(l => l.Warning(It.IsAny <Exception>(), It.IsAny <string>(), It.IsAny <object[]>()), Times.Never);
}
public async Task CheckpointManagerCreatesScope()
{
using ClientDiagnosticListener listener = new ClientDiagnosticListener(DiagnosticSourceName);
var eventHubName = "SomeName";
var endpoint = new Uri("amqp://some.endpoint.com/path");
Func <EventHubConnection> fakeFactory = () => new MockConnection(endpoint, eventHubName);
var context = new MockPartitionContext("partition");
var data = new MockEventData(new byte[0], sequenceNumber: 0, offset: 0);
var storageManager = new Mock <PartitionManager>();
var eventProcessor = new Mock <EventProcessorClient>(Mock.Of <BlobContainerClient>(), "cg", endpoint.Host, eventHubName, fakeFactory, null);
storageManager
.Setup(manager => manager.UpdateCheckpointAsync(It.IsAny <Checkpoint>()))
.Returns(Task.CompletedTask);
eventProcessor
.Setup(processor => processor.CreateStorageManager(It.IsAny <BlobContainerClient>()))
.Returns(storageManager.Object);
await eventProcessor.Object.UpdateCheckpointAsync(data, context);
ClientDiagnosticListener.ProducedDiagnosticScope scope = listener.Scopes.Single();
Assert.That(scope.Name, Is.EqualTo(DiagnosticProperty.EventProcessorCheckpointActivityName));
}
public async Task WhenReceivesNullEvents_ThenDoesNotFlushEmptyBuffer()
{
var operationQueue = new List <string>();
IList <BlockData> blocks = null;
Action <IReadOnlyCollection <BlockData>, CancellationToken> flagWriteOperation = (d, ct) =>
{
operationQueue.Add(write);
blocks = d.ToList();
};
Func <Task> flagCheckpointOperation = () =>
{
operationQueue.Add(checkpoint);
return(Task.FromResult(true));
};
var context = MockPartitionContext.Create("0", flagCheckpointOperation);
await _processor.OpenAsync(context);
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(true))
.Callback(flagWriteOperation);
await _processor.ProcessEventsAsync(context, null);
_writerMock.Verify(
w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()),
Times.Never());
// Assert checkpoint is never called
Assert.Empty(operationQueue);
}
public async Task ShouldNotPublishToPoisonQueueWhenMessageIsHandled()
{
var handled = false;
var context = MockPartitionContext.CreateWithNoopCheckpoint("1");
var events = new[] { new EventData() };
var resovler = new MockMessageHandlerResolver(
(body, headers) =>
{
handled = true;
return(Task.FromResult <object>(null));
});
var poisonHandler = (MockPoisonMessageHandler)DependencyResolverFactory
.GetResolver()
.GetService(typeof(IPoisonMessageHandler));
var processor = new EventProcessor(resovler, new MockCircuitBreaker(), 1, "test", Mock.Of <IDispatcherInstrumentationPublisher>());
await processor.ProcessEventsAsync(context, events);
Assert.True(handled);
Assert.False(
poisonHandler.Messages.Any(),
String.Format("Expected poison handler to have no messages; count = {0}",
poisonHandler.Messages.Count()));
}
public async Task ShouldProcessMessagesConcurrently()
{
const int Concurrency = 4;
const int MessageCount = 4;
const int MsPerMessage = 300;
var context = MockPartitionContext.CreateWithNoopCheckpoint("1");
var events = Enumerable
.Range(0, MessageCount)
.Select(id => new EventData())
.ToArray();
var resovler = new MockMessageHandlerResolver(
async(body, headers) => { await Task.Delay(TimeSpan.FromMilliseconds(MsPerMessage)); });
var processor = new EventProcessor(resovler, new MockCircuitBreaker(), Concurrency, "test", Mock.Of <IDispatcherInstrumentationPublisher>());
var sw = Stopwatch.StartNew();
await processor.ProcessEventsAsync(context, events);
sw.Stop();
Assert.True(sw.Elapsed < TimeSpan.FromMilliseconds(MsPerMessage * MessageCount));
Assert.True(sw.Elapsed >= TimeSpan.FromMilliseconds(MsPerMessage));
}
public async Task ShouldPublishToPoisonQueueWhenHandlerThrows()
{
var context = MockPartitionContext.CreateWithNoopCheckpoint("1");
var attemptedToResolveHandler = false;
var events = new[] { new EventData() };
var resolver = new MockMessageHandlerResolver(
async(body, headers) =>
{
attemptedToResolveHandler = true;
await Task.Yield();
throw new Exception("This message was bad.");
});
var poisonHandler = (MockPoisonMessageHandler)DependencyResolverFactory
.GetResolver()
.GetService(typeof(IPoisonMessageHandler));
var processor = new EventProcessor(resolver, new MockCircuitBreaker(), 1, "test", Mock.Of <IDispatcherInstrumentationPublisher>());
await processor.ProcessEventsAsync(context, events);
Assert.True(attemptedToResolveHandler);
Assert.True(
poisonHandler.Messages.Any(),
String.Format("Expected poison handler to have messages; count = {0}",
poisonHandler.Messages.Count()));
}
public async Task WhenUnableToCheckpointWithStorageExceptionThenLogs()
{
// Arrange
var context = MockPartitionContext.Create("0", () =>
{
throw new StorageException();
});
var processor =
new ColdStorageProcessor(
n => _writerMock.Object,
Mock.Of <IColdStorageInstrumentationPublisher>(),
CancellationToken.None,
CircuitBreakerWarningLevel,
CircuitBreakerStallLevel,
CircuitBreakerStallInterval,
TimeSpan.FromSeconds(200),
"Test",
maxBlockSize: MaxBlockSize);
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(() => TaskHelpers.CreateCompletedTask(true));
await processor.OpenAsync(context);
// Act
await processor.ProcessEventsAsync(context, new[]
{
CreateEventData((byte)'a', 100),
CreateEventData((byte)'b', 200),
CreateEventData((byte)'c', 300),
CreateEventData((byte)'d', 400),
});
}
public async Task WhenClosingForLostLeaseWithEmptyBuffer_ThenDoesThrow()
{
var operationQueue = new List <string>();
Func <Task> flagCheckpointOperation = () =>
{
operationQueue.Add(checkpoint);
return(Task.FromResult(true));
};
IList <BlockData> blocks = null;
Action <IReadOnlyCollection <BlockData>, CancellationToken> getBlocks = (d, ct) => blocks = d.ToList();
var context = MockPartitionContext.Create("0", flagCheckpointOperation);
await _processor.OpenAsync(context);
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(true))
.Callback(getBlocks);
await _processor.CloseAsync(context, CloseReason.LeaseLost);
_writerMock.Verify(
w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()),
Times.Never());
// Assert checkpoint is never called
Assert.Empty(operationQueue);
}
public async Task WhenIncomingMessagesFillABlock_ThenWritesFilledBlock()
{
// Arrange
var operationQueue = new List <string>();
IList <BlockData> blocks = null;
Action <IReadOnlyCollection <BlockData>, CancellationToken> flagWriteOperation = (d, ct) =>
{
operationQueue.Add(write);
blocks = d.ToList();
};
Func <Task> flagCheckpointOperation = () =>
{
operationQueue.Add(checkpoint);
return(Task.FromResult(true));
};
var context = MockPartitionContext.Create("0", flagCheckpointOperation);
await _processor.OpenAsync(context);
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(true))
.Callback(flagWriteOperation);
// Act
await _processor.ProcessEventsAsync(
context,
new[]
{
CreateEventData((byte)'a', 100),
CreateEventData((byte)'b', 200),
CreateEventData((byte)'c', 300),
CreateEventData((byte)'d', 400),
});
Assert.NotNull(blocks);
Assert.Equal(1, blocks.Count);
var serializedFrame = Encoding.UTF8.GetString(blocks[0].Frame, 0, blocks[0].FrameLength);
var lines = serializedFrame.Split(new[] { ColdStorageProcessor.EventDelimiter }, StringSplitOptions.RemoveEmptyEntries);
Assert.Equal(3, lines.Length);
Assert.Equal(new string('a', 100), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[0])).Payload);
Assert.Equal(new string('b', 200), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[1])).Payload);
Assert.Equal(new string('c', 300), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[2])).Payload);
// Assert checkpoint happens after write
Assert.Equal(write, operationQueue.FirstOrDefault());
Assert.Equal(checkpoint, operationQueue.LastOrDefault());
}
public async Task WhenCachedBlocksHitTripLevel_ThenStallsUntilWriteOperationSucceeds()
{
var blocks = new List <IList <BlockData> >();
Action <IReadOnlyCollection <BlockData>, CancellationToken> saveBlocks = (d, ct) => blocks.Add(d.ToList());
var failWrite = true;
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(() => TaskHelpers.CreateCompletedTask(!Volatile.Read(ref failWrite)))
.Callback(saveBlocks);
var context = MockPartitionContext.CreateWithNoopCheckpoint("0");
await _processor.OpenAsync(context);
// first N requests fail but go through (the first batch will not fill a frame so it won't result in a write operation)
for (int i = 0; i < CircuitBreakerStallLevel + 1; i++)
{
var batch = new[] { CreateEventData((byte)('a' + i), MaxBlockSize - 200 - i), };
Task processTask = _processor.ProcessEventsAsync(context, batch);
await AssertExt.CompletesBeforeTimeoutAsync(processTask, TimeoutInterval);
}
// N+1th stalls and waits for cached frames to be flushed
{
var batch = new[] { CreateEventData((byte)('a' + CircuitBreakerStallLevel + 2), 10), };
Task processTask = _processor.ProcessEventsAsync(context, batch);
await AssertExt.DoesNotCompleteBeforeTimeoutAsync(processTask, TimeoutInterval);
// let the write operation through
Volatile.Write(ref failWrite, false);
await AssertExt.CompletesBeforeTimeoutAsync(processTask, TimeoutInterval);
// check the stalled entries were written
var bufferedBlocks = blocks.Last();
Assert.Equal(CircuitBreakerStallLevel, bufferedBlocks.Count);
for (int i = 0; i < CircuitBreakerStallLevel; i++)
{
var lines = GetPayloadsFromBlock(bufferedBlocks[i]);
Assert.Equal(1, lines.Length);
Assert.Equal(
new string((char)('a' + i), MaxBlockSize - 200 - i),
lines[0]);
}
}
}
public async Task ProcessEventsAsyncSendsEventDataToWriter()
{
var context = MockPartitionContext.CreateWithNoopCheckpoint("0");
await _processor.OpenAsync(context);
var eventData = new EventData(Encoding.UTF8.GetBytes("{ \"property1\": \"value1\"}"));
var eventDataArray = new[]
{
eventData
};
await _processor.ProcessEventsAsync(context, eventDataArray);
_writerMock.Verify(
w => w.WriteAsync(It.Is <List <EventData> >(p => p.Contains(eventData)), It.IsAny <CancellationToken>()),
Times.Once);
}
public async Task WhenIncomingMessagesFailToFillABlock_ThenDoesNotWrite()
{
var context = MockPartitionContext.CreateWithNoopCheckpoint("0");
await _processor.OpenAsync(context);
await _processor.ProcessEventsAsync(
context,
new[]
{
CreateEventData((byte)'a', 100),
CreateEventData((byte)'b', 200),
CreateEventData((byte)'c', 300),
});
_writerMock.Verify(
w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()),
Times.Never());
}
public async Task BuildingIdPropertyPopulatedFromLookupService()
{
_lookupServiceMock.Setup(service => service.GetBuildingId("123")).Returns(() => "456");
var context = MockPartitionContext.CreateWithNoopCheckpoint("0");
await _processor.OpenAsync(context);
var eventData = new EventData(Encoding.UTF8.GetBytes("{ \"DeviceId\": \"123\"}"));
var eventDataArray = new[]
{
eventData
};
await _processor.ProcessEventsAsync(context, eventDataArray);
_writerMock.Verify(
w => w.WriteAsync(It.Is <List <EventData> >(p => p.First().Properties["BuildingId"] as string == "456"), It.IsAny <CancellationToken>()),
Times.Once);
}
public async Task WhenCachedBlocksRemainBelowTripLevel_ThenDoesNotStall()
{
var blocks = new List <IList <BlockData> >();
Action <IReadOnlyCollection <BlockData>, CancellationToken> saveBlocks = (d, ct) => blocks.Add(d.ToList());
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(() => TaskHelpers.CreateCompletedTask(false))
.Callback(saveBlocks);
var context = MockPartitionContext.CreateWithNoopCheckpoint("0");
await _processor.OpenAsync(context);
for (int i = 0; i < CircuitBreakerStallLevel; i++)
{
var batch = new[] { CreateEventData((byte)('a' + i), MaxBlockSize - 200 - i), };
Task processTask = _processor.ProcessEventsAsync(context, batch);
await AssertExt.CompletesBeforeTimeoutAsync(processTask, TimeoutInterval);
}
}
public async Task WhenWritingFullBlockFails_ThenDoesNotThrow()
{
// Arrange
var operationQueue = new List <string>();
Func <Task> flagCheckpointOperation = () =>
{
operationQueue.Add(checkpoint);
return(Task.FromResult(true));
};
IList <BlockData> blocks = null;
Action <IReadOnlyCollection <BlockData>, CancellationToken> getBlocks = (d, ct) => blocks = d.ToList();
var context = MockPartitionContext.Create("0", flagCheckpointOperation);
await _processor.OpenAsync(context);
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(() => TaskHelpers.CreateCompletedTask(false))
.Callback(getBlocks);
// Act
await _processor.ProcessEventsAsync(
context,
new[]
{
CreateEventData((byte)'a', 100),
CreateEventData((byte)'b', 200),
CreateEventData((byte)'c', 300),
CreateEventData((byte)'d', 400),
});
// Assert
Assert.NotNull(blocks);
Assert.Equal(1, blocks.Count);
Assert.False(operationQueue.Contains(checkpoint));
}
public async Task UpdateCheckpointAsyncCreatesScope()
{
using ClientDiagnosticListener listener = new ClientDiagnosticListener(DiagnosticSourceName);
var eventHubName = "SomeName";
var endpoint = new Uri("amqp://some.endpoint.com/path");
Func <EventHubConnection> fakeFactory = () => new MockConnection(endpoint, eventHubName);
var context = new MockPartitionContext("partition");
var data = new MockEventData(new byte[0], sequenceNumber: 0, offset: 0);
var storageManager = new Mock <PartitionManager>();
var eventProcessor = new Mock <EventProcessorClient>(Mock.Of <PartitionManager>(), "cg", endpoint.Host, eventHubName, fakeFactory, null);
// UpdateCheckpointAsync does not invoke the handlers, but we are setting them here in case
// this fact changes in the future.
eventProcessor.Object.ProcessEventAsync += eventArgs => Task.CompletedTask;
eventProcessor.Object.ProcessErrorAsync += eventArgs => Task.CompletedTask;
await eventProcessor.Object.UpdateCheckpointAsync(data, context, default);
ClientDiagnosticListener.ProducedDiagnosticScope scope = listener.Scopes.Single();
Assert.That(scope.Name, Is.EqualTo(DiagnosticProperty.EventProcessorCheckpointActivityName));
}
public void DoesNotThrowWhenCheckpointFailsWithLeaseLostExceptionTypes()
{
testDomain.DoCallBack(() =>
{
// Arrange
var mockLogger = new Mock <ILogger>();
LoggerFactory.Register(Mock.Of <ILogFactory>(f => f.Create(It.IsAny <string>()) == mockLogger.Object));
var mockCircuitBreaker = new MockCircuitBreaker();
var mockResolver = new MockMessageHandlerResolver();
var processor = new EventProcessor(mockResolver, mockCircuitBreaker, MaxConcurrency, EventHubName, Mock.Of <IDispatcherInstrumentationPublisher>());
Func <Task> faultedCheckpoint = () =>
{
throw new LeaseLostException();
};
var context = MockPartitionContext.Create(PartitionId, faultedCheckpoint);
var events = new[] { new EventData() };
// Act & Assert
processor.ProcessEventsAsync(context, events).Wait();
mockLogger.Verify(l => l.Warning(It.IsAny <Exception>(), It.IsAny <string>(), It.IsAny <object[]>()), Times.Once);
});
}
public async Task RunPartitionProcessingAsyncCreatesScopeForEventProcessing()
{
var mockStorage = new MockCheckPointStorage();
var mockConsumer = new Mock <EventHubConsumerClient>("cg", Mock.Of <EventHubConnection>(), default);
var mockProcessor = new Mock <EventProcessorClient>(mockStorage, "cg", "ns", "eh", Mock.Of <Func <EventHubConnection> >(), default)
{
CallBase = true
};
using ClientDiagnosticListener listener = new ClientDiagnosticListener(DiagnosticSourceName);
var completionSource = new TaskCompletionSource <object>(TaskCreationOptions.RunContinuationsAsynchronously);
var processEventCalls = 0;
mockConsumer
.Setup(consumer => consumer.ReadEventsFromPartitionAsync(
It.IsAny <string>(),
It.IsAny <EventPosition>(),
It.IsAny <ReadEventOptions>(),
It.IsAny <CancellationToken>()))
.Returns <string, EventPosition, ReadEventOptions, CancellationToken>((partitionId, position, options, token) =>
{
async IAsyncEnumerable <PartitionEvent> mockPartitionEventEnumerable()
{
var context = new MockPartitionContext(partitionId);
yield return(new PartitionEvent(context, new EventData(Array.Empty <byte>())
{
Properties = { { "Diagnostic-Id", "id" } }
}));
yield return(new PartitionEvent(context, new EventData(Array.Empty <byte>())
{
Properties = { { "Diagnostic-Id", "id2" } }
}));
while (!completionSource.Task.IsCompleted && !token.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(1));
yield return(new PartitionEvent());
}
yield break;
};
return(mockPartitionEventEnumerable());
});
mockProcessor
.Setup(processor => processor.CreateConsumer(
It.IsAny <string>(),
It.IsAny <EventHubConnection>(),
It.IsAny <EventHubConsumerClientOptions>()))
.Returns(mockConsumer.Object);
mockProcessor.Object.ProcessEventAsync += eventArgs =>
{
if (++processEventCalls == 2)
{
completionSource.SetResult(null);
}
return(Task.CompletedTask);
};
// RunPartitionProcessingAsync does not invoke the error handler, but we are setting it here in case
// this fact changes in the future.
mockProcessor.Object.ProcessErrorAsync += eventArgs => Task.CompletedTask;
// Start processing and wait for the consumer to be invoked. Set a cancellation for backup to ensure
// that the test completes deterministically.
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(15));
using var partitionProcessingTask = Task.Run(() => mockProcessor.Object.RunPartitionProcessingAsync("pid", default, cancellationSource.Token));
public async Task WhenWritingFullBlockFails_ThenReattemptsOnNextFilledBlock()
{
// Arrange
var operationQueue = new List <string>();
Func <Task> flagCheckpointOperation = () =>
{
operationQueue.Add(checkpoint);
return(Task.FromResult(true));
};
IList <BlockData> blocks = null;
Action <IReadOnlyCollection <BlockData>, CancellationToken> getBlocks = (d, ct) => blocks = d.ToList();
var context = MockPartitionContext.Create("0", flagCheckpointOperation);
await _processor.OpenAsync(context);
_writerMock
.Setup(w => w.WriteAsync(It.IsAny <IReadOnlyCollection <BlockData> >(), It.IsAny <CancellationToken>()))
.Returns(() => TaskHelpers.CreateCompletedTask(false))
.Callback(getBlocks);
// Act
await _processor.ProcessEventsAsync(
context,
new[]
{
CreateEventData((byte)'a', 100),
CreateEventData((byte)'b', 200),
CreateEventData((byte)'c', 300),
CreateEventData((byte)'d', 400),
});
Assert.NotNull(blocks);
Assert.Equal(1, blocks.Count);
Assert.False(operationQueue.Contains(checkpoint));
await _processor.ProcessEventsAsync(
context,
new[]
{
CreateEventData((byte)'e', 300),
CreateEventData((byte)'f', 500),
});
Assert.NotNull(blocks);
Assert.Equal(2, blocks.Count);
var serializedFrame = Encoding.UTF8.GetString(blocks[0].Frame, 0, blocks[0].FrameLength);
var lines = serializedFrame.Split(new[] { ColdStorageProcessor.EventDelimiter }, StringSplitOptions.RemoveEmptyEntries);
Assert.Equal(3, lines.Length);
Assert.Equal(new string('a', 100), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[0])).Payload);
Assert.Equal(new string('b', 200), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[1])).Payload);
Assert.Equal(new string('c', 300), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[2])).Payload);
serializedFrame = Encoding.UTF8.GetString(blocks[1].Frame, 0, blocks[1].FrameLength);
lines = serializedFrame.Split(new[] { ColdStorageProcessor.EventDelimiter }, StringSplitOptions.RemoveEmptyEntries);
Assert.Equal(2, lines.Length);
Assert.Equal(new string('d', 400), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[0])).Payload);
Assert.Equal(new string('e', 300), (JsonConvert.DeserializeObject <ColdStorageEvent>(lines[1])).Payload);
}
