public async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
{
const int setval = 5;
const int addval = 7;
const int expected = setval + addval;
const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions;
var faultInjectionControl = new FaultInjectionControl()
{
FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
};
List <IFaultInjectionTransactionTestGrain> grains =
Enumerable.Range(0, grainCount)
.Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
.ToList();
IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());
await coordinator.MultiGrainSet(grains, setval);
try
{
await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);
}
catch (OrleansTransactionException)
{
//if failed due to timeout or other legitimate transaction exception, try again. This should succeed
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
//if transactional state loaded correctly after reactivation, then following should pass
foreach (var grain in grains)
{
int actual = await grain.Get();
Assert.Equal(expected, actual);
}
}
public async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
{
const int setval = 5;
const int addval = 7;
int expected = setval + addval;
const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions;
var faultInjectionControl = new FaultInjectionControl()
{
FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
};
List <IFaultInjectionTransactionTestGrain> grains =
Enumerable.Range(0, grainCount)
.Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
.ToList();
IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());
await coordinator.MultiGrainSet(grains, setval);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
try
{
await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
}
catch (OrleansTransactionAbortedException)
{
// add delay between transactions so errors don't bleed into neighboring transactions
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
catch (OrleansTransactionException e)
{
this.output.WriteLine($"Call failed with exception: {e}, retrying without fault");
bool cascadingAbort = false;
bool firstAttempt = true;
do
{
cascadingAbort = false;
try
{
expected = await grains.First().Get() + addval;
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
catch (OrleansCascadingAbortException)
{
this.output.WriteLine($"Retry failed with OrleansCascadingAbortException: {e}, retrying without fault");
// should only encounter this when faulting after storage write
Assert.Equal(FaultInjectionType.ExceptionAfterStore, injectionType);
// only allow one retry
Assert.True(firstAttempt);
// add delay prevent castcading abort.
cascadingAbort = true;
firstAttempt = false;
}
} while (cascadingAbort);
}
//if transactional state loaded correctly after reactivation, then following should pass
foreach (var grain in grains)
{
int actual = await grain.Get();
Assert.Equal(expected, actual);
}
}
public void Reset()
{
this.FaultInjectionType = FaultInjectionType.None;
this.FaultInjectionPhase = TransactionFaultInjectPhase.None;
}
public async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
{
const int setval = 5;
const int addval = 7;
int expected = setval + addval;
const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions;
var faultInjectionControl = new FaultInjectionControl()
{
FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
};
List <IFaultInjectionTransactionTestGrain> grains =
Enumerable.Range(0, grainCount)
.Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
.ToList();
IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());
await coordinator.MultiGrainSet(grains, setval);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.BeforeConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
try
{
await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.BeforeConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
}
catch (OrleansTransactionAbortedException)
{
// add delay between transactions so errors don't bleed into neighboring transactions
// TODO : remove when slow slow abort is complete - jbragg
await Task.Delay(TimeSpan.FromSeconds(30));
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
catch (OrleansTransactionException)
{
// add delay between transactions so errors don't bleed into neighboring transactions
// TODO : remove when slow slow abort is complete - jbragg
await Task.Delay(TimeSpan.FromSeconds(30));
expected = await grains.First().Get() + addval;
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
//if transactional state loaded correctly after reactivation, then following should pass
foreach (var grain in grains)
{
int actual = await grain.Get();
Assert.Equal(expected, actual);
}
}
public override Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
{
return(base.MultiGrainWriteTransaction_FaultInjection(injectionPhase, injectionType));
}
