public FaultInjectionTransactionalState(TransactionalState <TState> txState, IControlledTransactionFaultInjector faultInjector, IGrainRuntime grainRuntime, ILogger <FaultInjectionTransactionalState <TState> > logger) { this.grainRuntime = grainRuntime; this.txState = txState; this.logger = logger; this.FaultInjectionControl = new FaultInjectionControl(); this.faultInjector = faultInjector; }
public FaultInjectionTransactionManager(IControlledTransactionFaultInjector faultInjector, FaultInjectionControl faultInjectionControl, TransactionManager <TState> tm, IGrainContext activationContext, ILogger logger, IGrainRuntime grainRuntime) { this.grainRuntime = grainRuntime; this.tm = tm; this.faultInjectionControl = faultInjectionControl; this.logger = logger; this.context = activationContext; this.faultInjector = faultInjector; }
public FaultInjectionTransactionalResource(IControlledTransactionFaultInjector faultInjector, FaultInjectionControl faultInjectionControl, TransactionalResource <TState> tResource, IGrainContext activationContext, ILogger logger, IGrainRuntime grainRuntime) { this.grainRuntime = grainRuntime; this.tResource = tResource; this.faultInjectionControl = faultInjectionControl; this.logger = logger; this.faultInjector = faultInjector; this.context = activationContext; }
public FaultInjectionTransactionalState(TransactionalState <TState> txState, IGrainActivationContext activationContext, IGrainRuntime grainRuntime, ILogger <FaultInjectionTransactionalState <TState> > logger) { this.grainRuntime = grainRuntime; this.txState = txState; this.logger = logger; this.FaultInjectionControl = new FaultInjectionControl(); //fault injector has to be injected to DI as a scoped service, so each grain activation share one injector. but different grain activation use different ones. this.faultInjector = activationContext.ActivationServices.GetService <IControlledTransactionFaultInjector>(); if (this.faultInjector == null) { throw new ArgumentOutOfRangeException($"Incorrect {nameof(faultInjector)} type configured. Only {nameof(IControlledTransactionFaultInjector)} is allowed."); } }
public Task Add(int numberToAdd, FaultInjectionControl faultInjectionControl = null) { //reset in case control from last tx isn't cleared for some reason this.data.FaultInjectionControl.Reset(); //dont replace it with this.data.FaultInjectionControl = faultInjectionControl, //this.data.FaultInjectionControl must remain the same reference if (faultInjectionControl != null) { this.data.FaultInjectionControl.FaultInjectionPhase = faultInjectionControl.FaultInjectionPhase; this.data.FaultInjectionControl.FaultInjectionType = faultInjectionControl.FaultInjectionType; } return(this.data.PerformUpdate(d => { this.logger.LogInformation($"Adding {numberToAdd} to value {d.Value}."); d.Value += numberToAdd; })); }
public Task MultiGrainAddAndFaultInjection(List <IFaultInjectionTransactionTestGrain> grains, int numberToAdd, FaultInjectionControl faultInjection = null) { return(Task.WhenAll(grains.Select(g => g.Add(numberToAdd, faultInjection)))); }
public virtual async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType) { const int setval = 5; const int addval = 7; int expected = setval + addval; const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions; var faultInjectionControl = new FaultInjectionControl() { FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType }; List <IFaultInjectionTransactionTestGrain> grains = Enumerable.Range(0, grainCount) .Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid())) .ToList(); IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid()); await coordinator.MultiGrainSet(grains, setval); // add delay between transactions so confirmation errors don't bleed into neighboring transactions if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm) { await Task.Delay(TimeSpan.FromSeconds(30)); } try { await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl); // add delay between transactions so confirmation errors don't bleed into neighboring transactions if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm) { await Task.Delay(TimeSpan.FromSeconds(30)); } } catch (OrleansTransactionAbortedException) { // add delay between transactions so errors don't bleed into neighboring transactions await coordinator.MultiGrainAddAndFaultInjection(grains, addval); } catch (OrleansTransactionException e) { this.testOutput($"Call failed with exception: {e}, retrying without fault"); bool cascadingAbort = false; bool firstAttempt = true; do { cascadingAbort = false; try { expected = await grains[0].Get() + addval; await coordinator.MultiGrainAddAndFaultInjection(grains, addval); } catch (OrleansCascadingAbortException) { this.testOutput($"Retry failed with OrleansCascadingAbortException: {e}, retrying without fault"); // should only encounter this when faulting after storage write injectionType.Should().Be(FaultInjectionType.ExceptionAfterStore); // only allow one retry firstAttempt.Should().BeTrue(); // add delay prevent castcading abort. cascadingAbort = true; firstAttempt = false; } } while (cascadingAbort); } //if transactional state loaded correctly after reactivation, then following should pass foreach (var grain in grains) { int actual = await grain.Get(); actual.Should().Be(expected); } }