public FaultInjectionTransactionalState(TransactionalState <TState> txState, IControlledTransactionFaultInjector faultInjector, IGrainRuntime grainRuntime, ILogger <FaultInjectionTransactionalState <TState> > logger)
 {
     this.grainRuntime          = grainRuntime;
     this.txState               = txState;
     this.logger                = logger;
     this.FaultInjectionControl = new FaultInjectionControl();
     this.faultInjector         = faultInjector;
 }
Exemplo n.º 2
0
 public FaultInjectionTransactionManager(IControlledTransactionFaultInjector faultInjector, FaultInjectionControl faultInjectionControl, TransactionManager <TState> tm, IGrainContext activationContext, ILogger logger, IGrainRuntime grainRuntime)
 {
     this.grainRuntime          = grainRuntime;
     this.tm                    = tm;
     this.faultInjectionControl = faultInjectionControl;
     this.logger                = logger;
     this.context               = activationContext;
     this.faultInjector         = faultInjector;
 }
Exemplo n.º 3
0
 public FaultInjectionTransactionalResource(IControlledTransactionFaultInjector faultInjector, FaultInjectionControl faultInjectionControl,
                                            TransactionalResource <TState> tResource, IGrainContext activationContext, ILogger logger, IGrainRuntime grainRuntime)
 {
     this.grainRuntime          = grainRuntime;
     this.tResource             = tResource;
     this.faultInjectionControl = faultInjectionControl;
     this.logger        = logger;
     this.faultInjector = faultInjector;
     this.context       = activationContext;
 }
Exemplo n.º 4
0
 public FaultInjectionTransactionalState(TransactionalState <TState> txState, IGrainActivationContext activationContext, IGrainRuntime grainRuntime, ILogger <FaultInjectionTransactionalState <TState> > logger)
 {
     this.grainRuntime          = grainRuntime;
     this.txState               = txState;
     this.logger                = logger;
     this.FaultInjectionControl = new FaultInjectionControl();
     //fault injector has to be injected to DI as a scoped service, so each grain activation share one injector. but different grain activation use different ones.
     this.faultInjector = activationContext.ActivationServices.GetService <IControlledTransactionFaultInjector>();
     if (this.faultInjector == null)
     {
         throw new ArgumentOutOfRangeException($"Incorrect {nameof(faultInjector)} type configured. Only {nameof(IControlledTransactionFaultInjector)} is allowed.");
     }
 }
        public Task Add(int numberToAdd, FaultInjectionControl faultInjectionControl = null)
        {
            //reset in case control from last tx isn't cleared for some reason
            this.data.FaultInjectionControl.Reset();
            //dont replace it with this.data.FaultInjectionControl = faultInjectionControl,
            //this.data.FaultInjectionControl must remain the same reference
            if (faultInjectionControl != null)
            {
                this.data.FaultInjectionControl.FaultInjectionPhase = faultInjectionControl.FaultInjectionPhase;
                this.data.FaultInjectionControl.FaultInjectionType  = faultInjectionControl.FaultInjectionType;
            }

            return(this.data.PerformUpdate(d =>
            {
                this.logger.LogInformation($"Adding {numberToAdd} to value {d.Value}.");
                d.Value += numberToAdd;
            }));
        }
Exemplo n.º 6
0
 public Task MultiGrainAddAndFaultInjection(List <IFaultInjectionTransactionTestGrain> grains, int numberToAdd,
                                            FaultInjectionControl faultInjection = null)
 {
     return(Task.WhenAll(grains.Select(g => g.Add(numberToAdd, faultInjection))));
 }
        public virtual async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
        {
            const int setval                = 5;
            const int addval                = 7;
            int       expected              = setval + addval;
            const int grainCount            = TransactionTestConstants.MaxCoordinatedTransactions;
            var       faultInjectionControl = new FaultInjectionControl()
            {
                FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
            };
            List <IFaultInjectionTransactionTestGrain> grains =
                Enumerable.Range(0, grainCount)
                .Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
                .ToList();

            IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());

            await coordinator.MultiGrainSet(grains, setval);

            // add delay between transactions so confirmation errors don't bleed into neighboring transactions
            if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm)
            {
                await Task.Delay(TimeSpan.FromSeconds(30));
            }
            try
            {
                await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);

                // add delay between transactions so confirmation errors don't bleed into neighboring transactions
                if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm)
                {
                    await Task.Delay(TimeSpan.FromSeconds(30));
                }
            }
            catch (OrleansTransactionAbortedException)
            {
                // add delay between transactions so errors don't bleed into neighboring transactions
                await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
            }
            catch (OrleansTransactionException e)
            {
                this.testOutput($"Call failed with exception: {e}, retrying without fault");
                bool cascadingAbort = false;
                bool firstAttempt   = true;

                do
                {
                    cascadingAbort = false;
                    try
                    {
                        expected = await grains[0].Get() + addval;
                        await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
                    }
                    catch (OrleansCascadingAbortException)
                    {
                        this.testOutput($"Retry failed with OrleansCascadingAbortException: {e}, retrying without fault");
                        // should only encounter this when faulting after storage write
                        injectionType.Should().Be(FaultInjectionType.ExceptionAfterStore);
                        // only allow one retry
                        firstAttempt.Should().BeTrue();
                        // add delay prevent castcading abort.
                        cascadingAbort = true;
                        firstAttempt   = false;
                    }
                } while (cascadingAbort);
            }

            //if transactional state loaded correctly after reactivation, then following should pass
            foreach (var grain in grains)
            {
                int actual = await grain.Get();

                actual.Should().Be(expected);
            }
        }