public void RuntimeInformationTest()
{
TestState state = new TestState();
state.Initialize("RuntimeInformation", 1, 0);
state.Options.EnableReceiverRuntimeMetric = true;
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1);
state.PrepareToRun();
state.StartRun(sfp);
state.RunForNBatches(20, 10);
state.WaitRun();
// EXPECTED RESULT: Normal processing. Verify that the RuntimeInformation property of the partition
// context is updated.
Assert.True(state.Processor.LatestContext.RuntimeInformation.LastSequenceNumber == state.Processor.LastEvent.SystemProperties.SequenceNumber);
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
}
public void CloseException()
{
TestState state = new TestState();
state.Initialize("CloseException", 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1);
state.PrepareToRun();
state.Processor.Injector =
new TestProcessor.ErrorInjector(TestProcessor.ErrorLocation.OnClose, new NotImplementedException("ErrorInjector"));
state.StartRun(sfp);
state.VerifyNormalStartup(10);
state.CountNBatches(20, 10);
state.TokenSource.Cancel();
// EXPECTED RESULT: Failure is traced but otherwise ignored.
state.WaitRun();
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
}
public void SmokeTest()
{
TestState state = new TestState();
state.Initialize("smoke", 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1);
state.PrepareToRun();
state.StartRun(sfp);
state.RunForNBatches(20, 10);
state.WaitRun();
// EXPECTED RESULT: Normal processing.
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
}
public void TimeoutSuppressTest()
{
TestState state = new TestState();
state.Initialize("timeoutsuppress", 1, 0);
state.Options.ReceiveTimeout = TimeSpan.FromSeconds(5.0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new TimeoutEventHubClientFactoryMock(1);
state.PrepareToRun();
state.StartRun(sfp);
state.VerifyNormalStartup(10);
Thread.Sleep(20000); // timeout is 5s, sleep 20s to allow some timeouts
state.DoNormalShutdown(10);
state.WaitRun();
// EXPECTED RESULT: Normal processing. Because calls on receive timeout are suppressed, batches should be 0.
Assert.True(state.Processor.TotalBatches == 0, $"ProcessEvents was called {state.Processor.TotalBatches} times");
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
}
public void UseInitialPositionProviderTest()
{
TestState state = new TestState();
state.Initialize("UseInitialPositionProvider", 1, 0);
const long firstSequenceNumber = 3456L;
state.Options.InitialPositionProvider = (partitionId) => { return EventPosition.FromSequenceNumber(firstSequenceNumber); };
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1);
state.PrepareToRun();
state.StartRun(sfp);
state.RunForNBatches(20, 10);
state.WaitRun();
// EXPECTED RESULT: Normal processing. Sequence number of first event processed should match that
// supplied in the InitialPositionProvider.
Assert.True(state.Processor.FirstEvent.SystemProperties.SequenceNumber == (firstSequenceNumber + 1L),
$"Got unexpected first sequence number {state.Processor.FirstEvent.SystemProperties.SequenceNumber}");
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
}
public void SimpleOptionsTest()
{
TestState state = new TestState();
state.Initialize("SimpleOptions", 1, 0);
const int testBatchSize = 42;
Assert.False(state.Options.MaxBatchSize == testBatchSize); // make sure new value is not the same as the default
state.Options.MaxBatchSize = testBatchSize;
const int testPrefetchCount = 444;
Assert.False(state.Options.PrefetchCount == testPrefetchCount);
state.Options.PrefetchCount = testPrefetchCount;
TimeSpan testReceiveTimeout = TimeSpan.FromSeconds(10.0);
Assert.False(state.Options.ReceiveTimeout.Equals(testReceiveTimeout));
state.Options.ReceiveTimeout = testReceiveTimeout;
ReceiverOptions receiverOptions = new ReceiverOptions();
Assert.Null(receiverOptions.Identifier);
const string tag = "SimpleOptions";
receiverOptions.Identifier = tag;
state.Options.ClientReceiverOptions = receiverOptions;
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1, tag);
state.PrepareToRun();
state.StartRun(sfp);
state.VerifyNormalStartup(10);
state.CountNBatches(5, 10);
// EXPECTED RESULT: Normal processing. Validate that simple options MaxBatchSize, PrefetchCount, ReceiveTimeout,
// and ClientReceiverOptions are passed through to EH API.
Assert.True(EventHubMocks.PartitionReceiverMock.receivers.ContainsKey(tag), "Cannot find receiver");
EventHubMocks.PartitionReceiverMock testReceiver = EventHubMocks.PartitionReceiverMock.receivers[tag];
Assert.True(testReceiver.HandlerBatchSize == testBatchSize, $"Unexpected batch size {testReceiver.HandlerBatchSize}");
Assert.True(testReceiver.PrefetchCount == testPrefetchCount, $"Unexpected prefetch count {testReceiver.PrefetchCount}");
Assert.True(testReceiver.ReceiveTimeout.Equals(testReceiveTimeout),
$"Unexpected receive timeout {testReceiver.ReceiveTimeout}");
Assert.NotNull(testReceiver.Options);
Assert.Equal(testReceiver.Options.Identifier, tag);
// EnableReceiverRuntimeMetric is false by default. This case is a convenient opportunity to
// verify that RuntimeInformation was not updated when the option is false.
Assert.True(state.Processor.LatestContext.RuntimeInformation.LastSequenceNumber == -1L,
$"RuntimeInformation.LastSequenceNumber is {state.Processor.LatestContext.RuntimeInformation.LastSequenceNumber}");
state.DoNormalShutdown(10);
state.WaitRun();
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
}
private void EventHubReceiveFailure(string name, Exception error, bool isEventHubsException)
{
TestState state = new TestState();
state.Initialize(name + "EventHubReceiveFailure", 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
NeverEHErrorInjector injector = new NeverEHErrorInjector(EHErrorLocation.Receiving, error);
sfp.EventHubClientFactory = new InjectorEventHubClientFactoryMock(1, injector);
state.PrepareToRun();
state.StartRun(sfp);
state.VerifyNormalStartup(10);
state.CountNBatches(20, 10);
Assert.True(EventHubMocks.PartitionReceiverMock.receivers.ContainsKey(InjectorEventHubClientFactoryMock.Tag),
"Cannot find receiver");
InjectorPartitionReceiverMock testReceiver = (InjectorPartitionReceiverMock)
EventHubMocks.PartitionReceiverMock.receivers[InjectorEventHubClientFactoryMock.Tag];
testReceiver.ForceReceiveError(error);
// EXPECTED RESULT: RunAsync will throw (Task completed exceptionally)
// due to nontransient EventHubsException or other exception type from EH operation.
// The Wait call bundles the exception into an AggregateException and rethrows.
state.OuterTask.Wait();
try
{
state.SFPTask.Wait();
}
catch (AggregateException ae)
{
Assert.True(ae.InnerExceptions.Count == 1, $"Unexpected number of errors {ae.InnerExceptions.Count}");
Exception inner = ae.InnerExceptions[0];
if (isEventHubsException)
{
Assert.True(inner is EventHubsException, $"Unexpected inner exception type {inner.GetType().Name}");
Assert.False(((EventHubsException)inner).IsTransient, "Inner exception is transient");
}
else
{
Assert.True(inner is Exception, $"Unexpected inner exception type {inner.GetType().Name}");
}
Assert.Contains("ErrorInjector", inner.Message);
}
}
public void TransientEventHubReceiveFailure()
{
TestState state = new TestState();
state.Initialize("TransientEventHubReceiveFailure", 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
EventHubsException error = new EventHubsException(true, "ErrorInjector");
NeverEHErrorInjector injector = new NeverEHErrorInjector(EHErrorLocation.Receiving, error);
sfp.EventHubClientFactory = new InjectorEventHubClientFactoryMock(1, injector);
state.PrepareToRun();
state.StartRun(sfp);
state.VerifyNormalStartup(10);
int batchesAlreadyDone = state.CountNBatches(20, 10);
Assert.True(EventHubMocks.PartitionReceiverMock.receivers.ContainsKey(InjectorEventHubClientFactoryMock.Tag),
"Cannot find receiver");
InjectorPartitionReceiverMock testReceiver = (InjectorPartitionReceiverMock)
EventHubMocks.PartitionReceiverMock.receivers[InjectorEventHubClientFactoryMock.Tag];
const int errorCount = 10;
for (int i = 0; i < errorCount; i++)
{
testReceiver.ForceReceiveError(error);
Thread.Sleep(100);
}
state.CountNBatches(batchesAlreadyDone * 2, 10);
state.DoNormalShutdown(10);
state.WaitRun();
// EXPECTED RESULT: Processing should happen normally but errors reported.
Assert.True(state.Processor.TotalBatches >= 20, $"Run ended at {state.Processor.TotalBatches} batches");
Assert.True(state.Processor.TotalErrors == errorCount, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
}
private void GeneralStartupFailure(string name, EHErrorLocation location, bool isEventHubsException)
{
TestState state = new TestState();
state.Initialize(name, 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
Exception injectee = isEventHubsException ? new EventHubsException(false, "ErrorInjector") :
new Exception("ErrorInjector");
OnceEHErrorInjector injector = new OnceEHErrorInjector(location, injectee);
sfp.EventHubClientFactory = new InjectorEventHubClientFactoryMock(1, injector);
state.PrepareToRun();
state.StartRun(sfp);
// EXPECTED RESULT: RunAsync will throw (Task completed exceptionally) during startup
// due to nontransient EventHubsException or other exception type from EH operation.
// The Wait call bundles the exception into an AggregateException and rethrows.
state.OuterTask.Wait();
try
{
state.SFPTask.Wait();
}
catch (AggregateException ae)
{
Assert.True(ae.InnerExceptions.Count == 1, $"Unexpected number of errors {ae.InnerExceptions.Count}");
Exception inner = ae.InnerExceptions[0];
if (isEventHubsException)
{
Assert.True(inner is EventHubsException, $"Unexpected inner exception type {inner.GetType().Name}");
Assert.False(((EventHubsException)inner).IsTransient, "Inner exception is transient");
}
else
{
Assert.True(inner is Exception, $"Unexpected inner exception type {inner.GetType().Name}");
}
Assert.Contains("ErrorInjector", inner.Message);
}
}
public void IgnoreInitialPositionProviderTest()
{
TestState state = new TestState();
state.Initialize("IgnoreInitialPositionProvider", 1, 0);
const long ippSequenceNumber = 3456L;
state.Options.InitialPositionProvider = (partitionId) => { return(EventPosition.FromSequenceNumber(ippSequenceNumber)); };
// Fake up a checkpoint using code borrowed from ReliableDictionaryCheckpointManager
IReliableDictionary <string, Dictionary <string, object> > store =
state.StateManager.GetOrAddAsync <IReliableDictionary <string, Dictionary <string, object> > >("EventProcessorCheckpointDictionary").Result;
const long checkpointSequenceNumber = 8888L;
Checkpoint fake = new Checkpoint((checkpointSequenceNumber * 100L).ToString(), checkpointSequenceNumber);
using (ITransaction tx = state.StateManager.CreateTransaction())
{
store.SetAsync(tx, "0", fake.ToDictionary(), TimeSpan.FromSeconds(5.0), CancellationToken.None).Wait();
tx.CommitAsync().Wait();
}
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1);
state.PrepareToRun();
state.StartRun(sfp);
state.RunForNBatches(20, 10);
state.WaitRun();
// EXPECTED RESULT: Normal processing. Sequence number of first event processed should match that
// supplied in the checkpoint, NOT the InitialPositionProvider.
Assert.True(state.Processor.FirstEvent.SystemProperties.SequenceNumber == (checkpointSequenceNumber + 1L),
$"Got unexpected first sequence number {state.Processor.FirstEvent.SystemProperties.SequenceNumber}");
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
cancellationToken.Register(() => _logger.LogInformation($"{nameof(RunAsync)} is being cancelled"));
try
{
await Execution
.ExecuteAsync(cancellationToken,
_logger, _serviceEventSource,
nameof(ReceiverService), Context.PartitionId.ToString(),
async ct =>
{
await _switch(cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
var options = new EventProcessorOptions
{
OnShutdown = OnShutdown,
MaxBatchSize = MaxMessageCount,
PrefetchCount = MaxMessageCount,
InitialPositionProvider = s =>
{
_logger.LogInformation("Using InitialPositionProvider for {s}", s);
return(_initialPositionProvider(s));
}
};
_logger.LogInformation("Create ServiceFabricProcessor with {ConsumerGroup}", _options.ConsumerGroup);
var processorService = new ServiceFabricProcessor(
Context.ServiceName,
Context.PartitionId,
StateManager,
Partition,
CreateProcessor(_options, _telemetryClient, _logger, _serviceEventSource, _eventHandlerCreator),
_options.ConnectionString,
_options.ConsumerGroup,
options);
await processorService.RunAsync(cancellationToken);
});
}
catch (FabricTransientException e)
{
_logger.LogError(e, nameof(ReceiverService) + "Exception .RunAsync for {PartitionId}", Context.PartitionId);
}
}
private void HardTransientStartupFailure(string name, EHErrorLocation location)
{
TestState state = new TestState();
state.Initialize("HardTransient" + name + "Failure", 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
EventHubsException injectee = new EventHubsException(true, "ErrorInjector");
AlwaysEHErrorInjector injector = new AlwaysEHErrorInjector(location, injectee);
sfp.EventHubClientFactory = new InjectorEventHubClientFactoryMock(1, injector);
state.PrepareToRun();
state.StartRun(sfp);
// EXPECTED RESULT: RunAsync will throw (Task completed exceptionally) during startup
// after running out of retries on an EH operation.
// The Wait call bundles the exception into an AggregateException and rethrows.
state.OuterTask.Wait();
try
{
state.SFPTask.Wait();
}
catch (AggregateException ae)
{
Assert.True(ae.InnerExceptions.Count == 1, $"Unexpected number of errors {ae.InnerExceptions.Count}");
Exception inner1 = ae.InnerExceptions[0];
Assert.True(inner1 is Exception, $"Unexpected inner exception type {inner1.GetType().Name}");
Assert.StartsWith("Out of retries ", inner1.Message);
Assert.NotNull(inner1.InnerException);
Exception inner2 = inner1.InnerException;
Assert.True(inner2 is EventHubsException, $"Unexpected inner exception type {inner2.GetType().Name}");
Assert.True(((EventHubsException)inner2).IsTransient, "Inner exception is not transient");
Assert.Equal("ErrorInjector", inner2.Message);
}
}
public void ErrorException()
{
TestState state = new TestState();
state.Initialize("EventException", 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1);
state.PrepareToRun();
// have to inject errors in events to cause error handler to be called
List <TestProcessor.ErrorLocation> locations =
new List <TestProcessor.ErrorLocation>()
{
TestProcessor.ErrorLocation.OnEvents, TestProcessor.ErrorLocation.OnError
};
state.Processor.Injector = new TestProcessor.ErrorInjector(locations, new NotImplementedException("ErrorInjector"));
state.StartRun(sfp);
state.RunForNBatches(20, 10);
// EXPECTED RESULT: Errors from ProcessEventsAsync are reported to ProcessErrorsAsync, but the
// errors in ProcessErrorsAsync are ignored. The test implementation of ProcessErrorsAsync increments
// the error count before throwing, so we can verify that ProcessErrorsAsync was called.
state.WaitRun();
Assert.True(state.Processor.TotalErrors == state.Processor.TotalBatches,
$"Unexpected error count got {state.Processor.TotalErrors} expected {state.Processor.TotalBatches}");
Assert.True(state.Processor.LastError is NotImplementedException,
$"Unexpected exception type {state.Processor.LastError.GetType().Name}");
Assert.Null(state.ShutdownException);
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
EventProcessorOptions options = new EventProcessorOptions();
options.OnShutdown = OnShutdown;
string eventHubConnectionString = "------------------------------EVENT HUB CONNECTION STRING GOES HERE ----------------------------";
ServiceFabricProcessor processorService = new ServiceFabricProcessor(this.Context.ServiceName, this.Context.PartitionId, this.StateManager, this.Partition,
new SampleEventProcessor(), eventHubConnectionString, "$Default", options);
Task processing = processorService.RunAsync(cancellationToken);
// If there is nothing else to do, application can simply await on the task here instead of polling keepgoing
while (this.keepgoing)
{
// Do other stuff here
Thread.Sleep(1000);
}
await processing;
// The await may throw if there was an error.
}
private void innerPartitionCountEnforcement(int servicePartitions, int eventHubPartitions)
{
TestState state = new TestState();
state.Initialize("partitioncountenforcement", servicePartitions, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(eventHubPartitions);
state.PrepareToRun();
state.StartRun(sfp);
int retries = 0;
while (!state.HasShutDown && (retries < 10))
{
Thread.Sleep(1000);
retries++;
}
// EXPECTED RESULT: EventProcessorConfigurationException, and IEventProcessor.OpenAsync is never called.
Assert.True(state.HasShutDown, $"Shutdown notification did not occur after {retries} seconds");
Assert.False(state.Processor.IsOpened, "Processor was opened");
Assert.NotNull(state.ShutdownException);
Assert.True(state.ShutdownException is EventProcessorConfigurationException,
$"Unexpected exception type {state.ShutdownException.GetType().Name}");
Assert.Equal($"Service partition count {servicePartitions} does not match event hub partition count {eventHubPartitions}",
state.ShutdownException.Message);
}
public void OpenException()
{
TestState state = new TestState();
state.Initialize("OpenException", 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1);
state.PrepareToRun();
state.Processor.Injector =
new TestProcessor.ErrorInjector(TestProcessor.ErrorLocation.OnOpen, new NotImplementedException("ErrorInjector"));
state.StartRun(sfp);
// EXPECTED RESULT: Failure during startup, Task returned by SFP.RunAsync completed exceptionally.
state.OuterTask.Wait();
try
{
state.SFPTask.Wait();
Assert.True(false, "No exception thrown");
}
catch (AggregateException ae)
{
Assert.True(ae.InnerExceptions.Count == 1, $"Unexpected number of errors {ae.InnerExceptions.Count}");
Exception inner = ae.InnerExceptions[0];
Assert.True(inner is NotImplementedException, $"Unexpected inner exception type {inner.GetType().Name}");
Assert.Equal("ErrorInjector", inner.Message);
}
}
public void TimeoutInvokeTest()
{
TestState state = new TestState();
state.Initialize("timeoutinvoke", 1, 0);
state.Options.ReceiveTimeout = TimeSpan.FromMilliseconds(10);
state.Options.InvokeProcessorAfterReceiveTimeout = true;
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options)
{
MockMode = state.PartitionLister,
EventHubClientFactory = new TimeoutEventHubClientFactoryMock(1)
};
state.PrepareToRun();
state.StartRun(sfp);
state.VerifyNormalStartup(10);
state.CountNBatches(1, 10);
state.DoNormalShutdown(10);
state.WaitRun();
// EXPECTED RESULT: Normal processing. Calls on receive timeout are enabled, so batches should be nonzero.
Assert.True(state.Processor.TotalBatches > 0, "ProcessEvents was not called");
Assert.True(state.Processor.TotalEvents == 0, $"ProcessEvents got {state.Processor.TotalEvents} events");
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
}
private void NoFailures(string name, EHErrorInjector injector)
{
TestState state = new TestState();
state.Initialize(name, 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new InjectorEventHubClientFactoryMock(1, injector);
state.PrepareToRun();
state.StartRun(sfp);
state.RunForNBatches(20, 10);
// EXPECTED RESULT: Processing should happen normally with no errors reported.
//
// 1) The error is transient, so it should be retried, and for "soft" errors the
// test harness only throws on the first call, so the retry will succeed.
//
// 2) Errors during shutdown are traced but ignored.
state.WaitRun();
Assert.True(state.Processor.TotalBatches >= 20, $"Run ended at {state.Processor.TotalBatches} batches");
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
}
public void EventException()
{
TestState state = new TestState();
state.Initialize("EventException", 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1);
state.PrepareToRun();
state.Processor.Injector =
new TestProcessor.ErrorInjector(TestProcessor.ErrorLocation.OnEvents, new NotImplementedException("ErrorInjector"));
state.StartRun(sfp);
state.RunForNBatches(20, 10);
// EXPECTED RESULT: Errors are reported to IEventProcessor.ProcessErrorsAsync but processing
// continues normally. There should be one error per batch.
state.WaitRun();
Assert.True(state.Processor.TotalErrors == state.Processor.TotalBatches,
$"Unexpected error count {state.Processor.TotalErrors}. Expected { state.Processor.TotalBatches }");
Assert.True(state.Processor.LastError is NotImplementedException,
$"Unexpected exception type {state.Processor.LastError.GetType().Name}. Expected { nameof(NotImplementedException) }");
Assert.Null(state.ShutdownException);
}
public void CheckpointEventTest()
{
TestState state = new TestState();
state.Initialize("checkpointing", 1, 0);
const long checkpointAt = 57L;
state.Processor = new CheckpointingProcessor(state.Options, checkpointAt);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1);
state.PrepareToRun();
state.StartRun(sfp);
state.RunForNBatches(20, 10);
state.WaitRun();
// EXPECTED RESULT: Normal processing. This case checkpoints specific events. Validate that the
// last event processed has a higher sequence number than the checkpointed event.
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
EventData checkpointedEvent = ((CheckpointingProcessor)state.Processor).CheckpointedEvent;
Assert.NotNull(checkpointedEvent);
Assert.True(checkpointedEvent.SystemProperties.SequenceNumber == checkpointAt,
$"Checkpointed event has seq {checkpointedEvent.SystemProperties.SequenceNumber}, expected {checkpointAt}");
EventData lastEvent = state.Processor.LastEvent;
Assert.NotNull(lastEvent);
Assert.True(lastEvent.SystemProperties.SequenceNumber > checkpointedEvent.SystemProperties.SequenceNumber,
$"Unexpected sequence number {lastEvent.SystemProperties.SequenceNumber}");
state.Processor = new CheckpointingProcessor(state.Options);
sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1);
state.PrepareToRun();
state.StartRun(sfp);
state.RunForNBatches(1, 10);
state.WaitRun();
// EXPECTED RESULT: Normal processing. The sequence number of the first event processed in
// this stage should be one higher than the sequence number of the event checkpointed in
// the previous stage.
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
EventData restartEvent = ((CheckpointingProcessor)state.Processor).FirstEvent;
Assert.NotNull(restartEvent);
Assert.True((restartEvent.SystemProperties.SequenceNumber - checkpointedEvent.SystemProperties.SequenceNumber) == 1,
$"Unexpected change in sequence number from {checkpointedEvent.SystemProperties.SequenceNumber} to {restartEvent.SystemProperties.SequenceNumber}");
}
public void CheckpointBatchTest()
{
TestState state = new TestState();
state.Initialize("checkpointing", 1, 0);
state.Processor = new CheckpointingProcessor(state.Options);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1);
state.PrepareToRun();
state.StartRun(sfp);
state.RunForNBatches(20, 10);
state.WaitRun();
// EXPECTED RESULT: Normal processing. Last event processed is also the end of the batch
// and hence the final checkpoint. Save for next stage validaiton.
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
EventData checkpointedEvent = state.Processor.LastEvent;
Assert.NotNull(checkpointedEvent);
Assert.True(checkpointedEvent.SystemProperties.SequenceNumber > 0L,
$"Unexpected sequence number {checkpointedEvent.SystemProperties.SequenceNumber}");
state.Processor = new CheckpointingProcessor(state.Options);
sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new EventHubMocks.EventHubClientFactoryMock(1);
state.PrepareToRun();
state.StartRun(sfp);
state.RunForNBatches(1, 10);
state.WaitRun();
// EXPECTED RESULT: Normal processing. The sequence number of the first event processed in
// this stage should be one higher than the sequence number of the last event processed in
// the previous stage.
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
EventData restartEvent = ((CheckpointingProcessor)state.Processor).FirstEvent;
Assert.NotNull(restartEvent);
Assert.True((restartEvent.SystemProperties.SequenceNumber - checkpointedEvent.SystemProperties.SequenceNumber) == 1,
$"Unexpected change in sequence number from {checkpointedEvent.SystemProperties.SequenceNumber} to {restartEvent.SystemProperties.SequenceNumber}");
}
