public FaultInjectionTransactionalState(TransactionalState <TState> txState, IControlledTransactionFaultInjector faultInjector, IGrainRuntime grainRuntime, ILogger <FaultInjectionTransactionalState <TState> > logger)
{
this.grainRuntime = grainRuntime;
this.txState = txState;
this.logger = logger;
this.FaultInjectionControl = new FaultInjectionControl();
this.faultInjector = faultInjector;
}
public FaultInjectionTransactionManager(IControlledTransactionFaultInjector faultInjector, FaultInjectionControl faultInjectionControl, TransactionManager <TState> tm, IGrainContext activationContext, ILogger logger, IGrainRuntime grainRuntime)
{
this.grainRuntime = grainRuntime;
this.tm = tm;
this.faultInjectionControl = faultInjectionControl;
this.logger = logger;
this.context = activationContext;
this.faultInjector = faultInjector;
}
public FaultInjectionTransactionalResource(IControlledTransactionFaultInjector faultInjector, FaultInjectionControl faultInjectionControl,
TransactionalResource <TState> tResource, IGrainContext activationContext, ILogger logger, IGrainRuntime grainRuntime)
{
this.grainRuntime = grainRuntime;
this.tResource = tResource;
this.faultInjectionControl = faultInjectionControl;
this.logger = logger;
this.faultInjector = faultInjector;
this.context = activationContext;
}
public FaultInjectionTransactionalState(TransactionalState <TState> txState, IGrainActivationContext activationContext, IGrainRuntime grainRuntime, ILogger <FaultInjectionTransactionalState <TState> > logger)
{
this.grainRuntime = grainRuntime;
this.txState = txState;
this.logger = logger;
this.FaultInjectionControl = new FaultInjectionControl();
//fault injector has to be injected to DI as a scoped service, so each grain activation share one injector. but different grain activation use different ones.
this.faultInjector = activationContext.ActivationServices.GetService <IControlledTransactionFaultInjector>();
if (this.faultInjector == null)
{
throw new ArgumentOutOfRangeException($"Incorrect {nameof(faultInjector)} type configured. Only {nameof(IControlledTransactionFaultInjector)} is allowed.");
}
}
public Task Add(int numberToAdd, FaultInjectionControl faultInjectionControl = null)
{
//reset in case control from last tx isn't cleared for some reason
this.data.FaultInjectionControl.Reset();
//dont replace it with this.data.FaultInjectionControl = faultInjectionControl,
//this.data.FaultInjectionControl must remain the same reference
if (faultInjectionControl != null)
{
this.data.FaultInjectionControl.FaultInjectionPhase = faultInjectionControl.FaultInjectionPhase;
this.data.FaultInjectionControl.FaultInjectionType = faultInjectionControl.FaultInjectionType;
}
return(this.data.PerformUpdate(d =>
{
this.logger.LogInformation($"Adding {numberToAdd} to value {d.Value}.");
d.Value += numberToAdd;
}));
}
public Task MultiGrainAddAndFaultInjection(List <IFaultInjectionTransactionTestGrain> grains, int numberToAdd,
FaultInjectionControl faultInjection = null)
{
return(Task.WhenAll(grains.Select(g => g.Add(numberToAdd, faultInjection))));
}
public virtual async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
{
const int setval = 5;
const int addval = 7;
int expected = setval + addval;
const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions;
var faultInjectionControl = new FaultInjectionControl()
{
FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
};
List <IFaultInjectionTransactionTestGrain> grains =
Enumerable.Range(0, grainCount)
.Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
.ToList();
IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());
await coordinator.MultiGrainSet(grains, setval);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
try
{
await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
}
catch (OrleansTransactionAbortedException)
{
// add delay between transactions so errors don't bleed into neighboring transactions
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
catch (OrleansTransactionException e)
{
this.testOutput($"Call failed with exception: {e}, retrying without fault");
bool cascadingAbort = false;
bool firstAttempt = true;
do
{
cascadingAbort = false;
try
{
expected = await grains[0].Get() + addval;
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
catch (OrleansCascadingAbortException)
{
this.testOutput($"Retry failed with OrleansCascadingAbortException: {e}, retrying without fault");
// should only encounter this when faulting after storage write
injectionType.Should().Be(FaultInjectionType.ExceptionAfterStore);
// only allow one retry
firstAttempt.Should().BeTrue();
// add delay prevent castcading abort.
cascadingAbort = true;
firstAttempt = false;
}
} while (cascadingAbort);
}
//if transactional state loaded correctly after reactivation, then following should pass
foreach (var grain in grains)
{
int actual = await grain.Get();
actual.Should().Be(expected);
}
}
