private void EventHubReceiveFailure(string name, Exception error, bool isEventHubsException)
{
TestState state = new TestState();
state.Initialize(name + "EventHubReceiveFailure", 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
NeverEHErrorInjector injector = new NeverEHErrorInjector(EHErrorLocation.Receiving, error);
sfp.EventHubClientFactory = new InjectorEventHubClientFactoryMock(1, injector);
state.PrepareToRun();
state.StartRun(sfp);
state.VerifyNormalStartup(10);
state.CountNBatches(20, 10);
Assert.True(EventHubMocks.PartitionReceiverMock.receivers.ContainsKey(InjectorEventHubClientFactoryMock.Tag),
"Cannot find receiver");
InjectorPartitionReceiverMock testReceiver = (InjectorPartitionReceiverMock)
EventHubMocks.PartitionReceiverMock.receivers[InjectorEventHubClientFactoryMock.Tag];
testReceiver.ForceReceiveError(error);
// EXPECTED RESULT: RunAsync will throw (Task completed exceptionally)
// due to nontransient EventHubsException or other exception type from EH operation.
// The Wait call bundles the exception into an AggregateException and rethrows.
state.OuterTask.Wait();
try
{
state.SFPTask.Wait();
}
catch (AggregateException ae)
{
Assert.True(ae.InnerExceptions.Count == 1, $"Unexpected number of errors {ae.InnerExceptions.Count}");
Exception inner = ae.InnerExceptions[0];
if (isEventHubsException)
{
Assert.True(inner is EventHubsException, $"Unexpected inner exception type {inner.GetType().Name}");
Assert.False(((EventHubsException)inner).IsTransient, "Inner exception is transient");
}
else
{
Assert.True(inner is Exception, $"Unexpected inner exception type {inner.GetType().Name}");
}
Assert.Contains("ErrorInjector", inner.Message);
}
}
private void GeneralStartupFailure(string name, EHErrorLocation location, bool isEventHubsException)
{
TestState state = new TestState();
state.Initialize(name, 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
Exception injectee = isEventHubsException ? new EventHubsException(false, "ErrorInjector") :
new Exception("ErrorInjector");
OnceEHErrorInjector injector = new OnceEHErrorInjector(location, injectee);
sfp.EventHubClientFactory = new InjectorEventHubClientFactoryMock(1, injector);
state.PrepareToRun();
state.StartRun(sfp);
// EXPECTED RESULT: RunAsync will throw (Task completed exceptionally) during startup
// due to nontransient EventHubsException or other exception type from EH operation.
// The Wait call bundles the exception into an AggregateException and rethrows.
state.OuterTask.Wait();
try
{
state.SFPTask.Wait();
}
catch (AggregateException ae)
{
Assert.True(ae.InnerExceptions.Count == 1, $"Unexpected number of errors {ae.InnerExceptions.Count}");
Exception inner = ae.InnerExceptions[0];
if (isEventHubsException)
{
Assert.True(inner is EventHubsException, $"Unexpected inner exception type {inner.GetType().Name}");
Assert.False(((EventHubsException)inner).IsTransient, "Inner exception is transient");
}
else
{
Assert.True(inner is Exception, $"Unexpected inner exception type {inner.GetType().Name}");
}
Assert.Contains("ErrorInjector", inner.Message);
}
}
public void TransientEventHubReceiveFailure()
{
TestState state = new TestState();
state.Initialize("TransientEventHubReceiveFailure", 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
EventHubsException error = new EventHubsException(true, "ErrorInjector");
NeverEHErrorInjector injector = new NeverEHErrorInjector(EHErrorLocation.Receiving, error);
sfp.EventHubClientFactory = new InjectorEventHubClientFactoryMock(1, injector);
state.PrepareToRun();
state.StartRun(sfp);
state.VerifyNormalStartup(10);
int batchesAlreadyDone = state.CountNBatches(20, 10);
Assert.True(EventHubMocks.PartitionReceiverMock.receivers.ContainsKey(InjectorEventHubClientFactoryMock.Tag),
"Cannot find receiver");
InjectorPartitionReceiverMock testReceiver = (InjectorPartitionReceiverMock)
EventHubMocks.PartitionReceiverMock.receivers[InjectorEventHubClientFactoryMock.Tag];
const int errorCount = 10;
for (int i = 0; i < errorCount; i++)
{
testReceiver.ForceReceiveError(error);
Thread.Sleep(100);
}
state.CountNBatches(batchesAlreadyDone * 2, 10);
state.DoNormalShutdown(10);
state.WaitRun();
// EXPECTED RESULT: Processing should happen normally but errors reported.
Assert.True(state.Processor.TotalBatches >= 20, $"Run ended at {state.Processor.TotalBatches} batches");
Assert.True(state.Processor.TotalErrors == errorCount, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
}
private void HardTransientStartupFailure(string name, EHErrorLocation location)
{
TestState state = new TestState();
state.Initialize("HardTransient" + name + "Failure", 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
EventHubsException injectee = new EventHubsException(true, "ErrorInjector");
AlwaysEHErrorInjector injector = new AlwaysEHErrorInjector(location, injectee);
sfp.EventHubClientFactory = new InjectorEventHubClientFactoryMock(1, injector);
state.PrepareToRun();
state.StartRun(sfp);
// EXPECTED RESULT: RunAsync will throw (Task completed exceptionally) during startup
// after running out of retries on an EH operation.
// The Wait call bundles the exception into an AggregateException and rethrows.
state.OuterTask.Wait();
try
{
state.SFPTask.Wait();
}
catch (AggregateException ae)
{
Assert.True(ae.InnerExceptions.Count == 1, $"Unexpected number of errors {ae.InnerExceptions.Count}");
Exception inner1 = ae.InnerExceptions[0];
Assert.True(inner1 is Exception, $"Unexpected inner exception type {inner1.GetType().Name}");
Assert.StartsWith("Out of retries ", inner1.Message);
Assert.NotNull(inner1.InnerException);
Exception inner2 = inner1.InnerException;
Assert.True(inner2 is EventHubsException, $"Unexpected inner exception type {inner2.GetType().Name}");
Assert.True(((EventHubsException)inner2).IsTransient, "Inner exception is not transient");
Assert.Equal("ErrorInjector", inner2.Message);
}
}
private void NoFailures(string name, EHErrorInjector injector)
{
TestState state = new TestState();
state.Initialize(name, 1, 0);
ServiceFabricProcessor sfp = new ServiceFabricProcessor(
state.ServiceUri,
state.ServicePartitionId,
state.StateManager,
state.StatefulServicePartition,
state.Processor,
state.ConnectionString,
"$Default",
state.Options);
sfp.MockMode = state.PartitionLister;
sfp.EventHubClientFactory = new InjectorEventHubClientFactoryMock(1, injector);
state.PrepareToRun();
state.StartRun(sfp);
state.RunForNBatches(20, 10);
// EXPECTED RESULT: Processing should happen normally with no errors reported.
//
// 1) The error is transient, so it should be retried, and for "soft" errors the
// test harness only throws on the first call, so the retry will succeed.
//
// 2) Errors during shutdown are traced but ignored.
state.WaitRun();
Assert.True(state.Processor.TotalBatches >= 20, $"Run ended at {state.Processor.TotalBatches} batches");
Assert.True(state.Processor.TotalErrors == 0, $"Errors found {state.Processor.TotalErrors}");
Assert.Null(state.ShutdownException);
}
