public Task Add(int numberToAdd, FaultInjectionControl faultInjectionControl = null)
{
//dont replace it with this.data.FaultInjectionControl = faultInjectionControl,
//this.data.FaultInjectionControl must remain the same reference
if (faultInjectionControl == null)
{
this.data.FaultInjectionControl.FaultInjectionPhase = TransactionFaultInjectPhase.None;
this.data.FaultInjectionControl.FaultInjectionType = FaultInjectionType.None;
}
else
{
this.data.FaultInjectionControl.FaultInjectionPhase = faultInjectionControl.FaultInjectionPhase;
this.data.FaultInjectionControl.FaultInjectionType = faultInjectionControl.FaultInjectionType;
}
return(this.data.PerformUpdate(d => d.Value += numberToAdd));
}
public Task Add(int numberToAdd, FaultInjectionControl faultInjectionControl = null)
{
//reset in case control from last tx isn't cleared for some reason
this.data.FaultInjectionControl.Reset();
//dont replace it with this.data.FaultInjectionControl = faultInjectionControl,
//this.data.FaultInjectionControl must remain the same reference
if (faultInjectionControl != null)
{
this.data.FaultInjectionControl.FaultInjectionPhase = faultInjectionControl.FaultInjectionPhase;
this.data.FaultInjectionControl.FaultInjectionType = faultInjectionControl.FaultInjectionType;
}
return(this.data.PerformUpdate(d =>
{
this.logger.LogInformation($"Adding {numberToAdd} to value {d.Value}.");
d.Value += numberToAdd;
}));
}
public async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
{
const int setval = 5;
const int addval = 7;
const int expected = setval + addval;
const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions;
var faultInjectionControl = new FaultInjectionControl()
{
FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
};
List <IFaultInjectionTransactionTestGrain> grains =
Enumerable.Range(0, grainCount)
.Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
.ToList();
IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());
await coordinator.MultiGrainSet(grains, setval);
try
{
await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);
}
catch (OrleansTransactionException)
{
//if failed due to timeout or other legitimate transaction exception, try again. This should succeed
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
//if transactional state loaded correctly after reactivation, then following should pass
foreach (var grain in grains)
{
int actual = await grain.Get();
Assert.Equal(expected, actual);
}
}
public async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
{
const int setval = 5;
const int addval = 7;
int expected = setval + addval;
const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions;
var faultInjectionControl = new FaultInjectionControl()
{
FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
};
List <IFaultInjectionTransactionTestGrain> grains =
Enumerable.Range(0, grainCount)
.Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
.ToList();
IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());
await coordinator.MultiGrainSet(grains, setval);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
try
{
await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
}
catch (OrleansTransactionAbortedException)
{
// add delay between transactions so errors don't bleed into neighboring transactions
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
catch (OrleansTransactionException e)
{
this.output.WriteLine($"Call failed with exception: {e}, retrying without fault");
bool cascadingAbort = false;
bool firstAttempt = true;
do
{
cascadingAbort = false;
try
{
expected = await grains.First().Get() + addval;
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
catch (OrleansCascadingAbortException)
{
this.output.WriteLine($"Retry failed with OrleansCascadingAbortException: {e}, retrying without fault");
// should only encounter this when faulting after storage write
Assert.Equal(FaultInjectionType.ExceptionAfterStore, injectionType);
// only allow one retry
Assert.True(firstAttempt);
// add delay prevent castcading abort.
cascadingAbort = true;
firstAttempt = false;
}
} while (cascadingAbort);
}
//if transactional state loaded correctly after reactivation, then following should pass
foreach (var grain in grains)
{
int actual = await grain.Get();
Assert.Equal(expected, actual);
}
}
public async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
{
const int setval = 5;
const int addval = 7;
int expected = setval + addval;
const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions;
var faultInjectionControl = new FaultInjectionControl()
{
FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
};
List <IFaultInjectionTransactionTestGrain> grains =
Enumerable.Range(0, grainCount)
.Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
.ToList();
IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());
await coordinator.MultiGrainSet(grains, setval);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.BeforeConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
try
{
await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.BeforeConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
}
catch (OrleansTransactionAbortedException)
{
// add delay between transactions so errors don't bleed into neighboring transactions
// TODO : remove when slow slow abort is complete - jbragg
await Task.Delay(TimeSpan.FromSeconds(30));
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
catch (OrleansTransactionException)
{
// add delay between transactions so errors don't bleed into neighboring transactions
// TODO : remove when slow slow abort is complete - jbragg
await Task.Delay(TimeSpan.FromSeconds(30));
expected = await grains.First().Get() + addval;
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
//if transactional state loaded correctly after reactivation, then following should pass
foreach (var grain in grains)
{
int actual = await grain.Get();
Assert.Equal(expected, actual);
}
}
public Task MultiGrainAddAndFaultInjection(List <IFaultInjectionTransactionTestGrain> grains, int numberToAdd,
FaultInjectionControl faultInjection = null)
{
return(Task.WhenAll(grains.Select(g => g.Add(numberToAdd, faultInjection))));
}
