public async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
        {
            const int setval                = 5;
            const int addval                = 7;
            const int expected              = setval + addval;
            const int grainCount            = TransactionTestConstants.MaxCoordinatedTransactions;
            var       faultInjectionControl = new FaultInjectionControl()
            {
                FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
            };
            List <IFaultInjectionTransactionTestGrain> grains =
                Enumerable.Range(0, grainCount)
                .Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
                .ToList();

            IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());

            await coordinator.MultiGrainSet(grains, setval);

            try
            {
                await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);
            }
            catch (OrleansTransactionException)
            {
                //if failed due to timeout or other legitimate transaction exception, try again. This should succeed
                await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
            }

            //if transactional state loaded correctly after reactivation, then following should pass
            foreach (var grain in grains)
            {
                int actual = await grain.Get();

                Assert.Equal(expected, actual);
            }
        }
        public async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
        {
            const int setval                = 5;
            const int addval                = 7;
            int       expected              = setval + addval;
            const int grainCount            = TransactionTestConstants.MaxCoordinatedTransactions;
            var       faultInjectionControl = new FaultInjectionControl()
            {
                FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
            };
            List <IFaultInjectionTransactionTestGrain> grains =
                Enumerable.Range(0, grainCount)
                .Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
                .ToList();

            IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());

            await coordinator.MultiGrainSet(grains, setval);

            // add delay between transactions so confirmation errors don't bleed into neighboring transactions
            if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm)
            {
                await Task.Delay(TimeSpan.FromSeconds(30));
            }
            try
            {
                await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);

                // add delay between transactions so confirmation errors don't bleed into neighboring transactions
                if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm)
                {
                    await Task.Delay(TimeSpan.FromSeconds(30));
                }
            }
            catch (OrleansTransactionAbortedException)
            {
                // add delay between transactions so errors don't bleed into neighboring transactions
                await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
            }
            catch (OrleansTransactionException e)
            {
                this.output.WriteLine($"Call failed with exception: {e}, retrying without fault");
                bool cascadingAbort = false;
                bool firstAttempt   = true;

                do
                {
                    cascadingAbort = false;
                    try
                    {
                        expected = await grains.First().Get() + addval;

                        await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
                    }
                    catch (OrleansCascadingAbortException)
                    {
                        this.output.WriteLine($"Retry failed with OrleansCascadingAbortException: {e}, retrying without fault");
                        // should only encounter this when faulting after storage write
                        Assert.Equal(FaultInjectionType.ExceptionAfterStore, injectionType);
                        // only allow one retry
                        Assert.True(firstAttempt);
                        // add delay prevent castcading abort.
                        cascadingAbort = true;
                        firstAttempt   = false;
                    }
                } while (cascadingAbort);
            }

            //if transactional state loaded correctly after reactivation, then following should pass
            foreach (var grain in grains)
            {
                int actual = await grain.Get();

                Assert.Equal(expected, actual);
            }
        }
        public async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
        {
            const int setval                = 5;
            const int addval                = 7;
            int       expected              = setval + addval;
            const int grainCount            = TransactionTestConstants.MaxCoordinatedTransactions;
            var       faultInjectionControl = new FaultInjectionControl()
            {
                FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
            };
            List <IFaultInjectionTransactionTestGrain> grains =
                Enumerable.Range(0, grainCount)
                .Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
                .ToList();

            IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());

            await coordinator.MultiGrainSet(grains, setval);

            // add delay between transactions so confirmation errors don't bleed into neighboring transactions
            if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.BeforeConfirm)
            {
                await Task.Delay(TimeSpan.FromSeconds(30));
            }
            try
            {
                await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);

                // add delay between transactions so confirmation errors don't bleed into neighboring transactions
                if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.BeforeConfirm)
                {
                    await Task.Delay(TimeSpan.FromSeconds(30));
                }
            }
            catch (OrleansTransactionAbortedException)
            {
                // add delay between transactions so errors don't bleed into neighboring transactions
                // TODO : remove when slow slow abort is complete - jbragg
                await Task.Delay(TimeSpan.FromSeconds(30));

                await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
            }
            catch (OrleansTransactionException)
            {
                // add delay between transactions so errors don't bleed into neighboring transactions
                // TODO : remove when slow slow abort is complete - jbragg
                await Task.Delay(TimeSpan.FromSeconds(30));

                expected = await grains.First().Get() + addval;

                await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
            }

            //if transactional state loaded correctly after reactivation, then following should pass
            foreach (var grain in grains)
            {
                int actual = await grain.Get();

                Assert.Equal(expected, actual);
            }
        }