protected bool ReadFaultInjectionType(ref string line, ref FaultInjectionType faultInjectionType)
{
string nextLexem;
for (;;)
{
bool flag = false;
nextLexem = this.GetNextLexem(ref line, ref flag);
if (flag)
{
break;
}
if (string.IsNullOrEmpty(nextLexem))
{
return(true);
}
if (!(nextLexem == ","))
{
goto Block_2;
}
}
this.ReportError("Low level scanning error in the trace types statement.", new object[0]);
return(false);
Block_2:
if (nextLexem == ":")
{
this.ReportError("Bogus colon in the trace types statement.", new object[0]);
return(false);
}
faultInjectionType = this.GetFaultInjectionTypeByName(nextLexem);
return(true);
}
public async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
{
const int setval = 5;
const int addval = 7;
int expected = setval + addval;
const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions;
var faultInjectionControl = new FaultInjectionControl()
{
FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
};
List <IFaultInjectionTransactionTestGrain> grains =
Enumerable.Range(0, grainCount)
.Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
.ToList();
IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());
await coordinator.MultiGrainSet(grains, setval);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
try
{
await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.AfterConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
}
catch (OrleansTransactionAbortedException)
{
// add delay between transactions so errors don't bleed into neighboring transactions
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
catch (OrleansTransactionException e)
{
this.output.WriteLine($"Call failed with exception: {e}, retrying without fault");
bool cascadingAbort = false;
bool firstAttempt = true;
do
{
cascadingAbort = false;
try
{
expected = await grains.First().Get() + addval;
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
catch (OrleansCascadingAbortException)
{
this.output.WriteLine($"Retry failed with OrleansCascadingAbortException: {e}, retrying without fault");
// should only encounter this when faulting after storage write
Assert.Equal(FaultInjectionType.ExceptionAfterStore, injectionType);
// only allow one retry
Assert.True(firstAttempt);
// add delay prevent castcading abort.
cascadingAbort = true;
firstAttempt = false;
}
} while (cascadingAbort);
}
//if transactional state loaded correctly after reactivation, then following should pass
foreach (var grain in grains)
{
int actual = await grain.Get();
Assert.Equal(expected, actual);
}
}
public async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
{
const int setval = 5;
const int addval = 7;
const int expected = setval + addval;
const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions;
var faultInjectionControl = new FaultInjectionControl()
{
FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
};
List <IFaultInjectionTransactionTestGrain> grains =
Enumerable.Range(0, grainCount)
.Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
.ToList();
IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());
await coordinator.MultiGrainSet(grains, setval);
try
{
await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);
}
catch (OrleansTransactionException)
{
//if failed due to timeout or other legitimate transaction exception, try again. This should succeed
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
//if transactional state loaded correctly after reactivation, then following should pass
foreach (var grain in grains)
{
int actual = await grain.Get();
Assert.Equal(expected, actual);
}
}
public async Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
{
const int setval = 5;
const int addval = 7;
int expected = setval + addval;
const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions;
var faultInjectionControl = new FaultInjectionControl()
{
FaultInjectionPhase = injectionPhase, FaultInjectionType = injectionType
};
List <IFaultInjectionTransactionTestGrain> grains =
Enumerable.Range(0, grainCount)
.Select(i => this.grainFactory.GetGrain <IFaultInjectionTransactionTestGrain>(Guid.NewGuid()))
.ToList();
IFaultInjectionTransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain <IFaultInjectionTransactionCoordinatorGrain>(Guid.NewGuid());
await coordinator.MultiGrainSet(grains, setval);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.BeforeConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
try
{
await coordinator.MultiGrainAddAndFaultInjection(grains, addval, faultInjectionControl);
// add delay between transactions so confirmation errors don't bleed into neighboring transactions
if (injectionPhase == TransactionFaultInjectPhase.BeforeConfirm || injectionPhase == TransactionFaultInjectPhase.BeforeConfirm)
{
await Task.Delay(TimeSpan.FromSeconds(30));
}
}
catch (OrleansTransactionAbortedException)
{
// add delay between transactions so errors don't bleed into neighboring transactions
// TODO : remove when slow slow abort is complete - jbragg
await Task.Delay(TimeSpan.FromSeconds(30));
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
catch (OrleansTransactionException)
{
// add delay between transactions so errors don't bleed into neighboring transactions
// TODO : remove when slow slow abort is complete - jbragg
await Task.Delay(TimeSpan.FromSeconds(30));
expected = await grains.First().Get() + addval;
await coordinator.MultiGrainAddAndFaultInjection(grains, addval);
}
//if transactional state loaded correctly after reactivation, then following should pass
foreach (var grain in grains)
{
int actual = await grain.Get();
Assert.Equal(expected, actual);
}
}
protected void ProcessFaultInjectionLine(string line)
{
InternalBypassTrace.FaultInjectionConfigurationTracer.TraceDebug(50735, 0L, "Processing line: {0}", new object[]
{
line
});
string text = null;
FaultInjectionType type = FaultInjectionType.None;
string empty = string.Empty;
uint key = 0U;
List <string> parameters = null;
for (;;)
{
bool flag = false;
string nextLexem = this.GetNextLexem(ref line, ref flag);
if (flag)
{
break;
}
if (string.IsNullOrEmpty(nextLexem))
{
goto IL_A4;
}
if (nextLexem == ",")
{
if (!this.ReadFaultInjectionTagComponent(ref line, ref empty))
{
goto Block_4;
}
}
else
{
if (nextLexem == ":")
{
goto IL_A4;
}
text = nextLexem;
}
}
this.ReportError("Low level scanning error in the statement header.", new object[0]);
return;
Block_4:
this.ReportError("Failed to read fault injection tag component.", new object[0]);
return;
IL_A4:
if (string.IsNullOrEmpty(text))
{
this.ReportError("Colon is not preceded with component name.", new object[0]);
return;
}
TraceComponentInfo traceComponentInfo = this.CreateComponentIfNeccessary(text, true);
if (traceComponentInfo == null)
{
this.ReportError("Not a component.", new object[0]);
return;
}
if (!this.ReadFaultInjectionType(ref line, ref type))
{
this.ReportError("Failed to read fault injection type.", new object[0]);
return;
}
if (!this.ReadFaultInjectionLid(ref line, ref key))
{
this.ReportError("Failed to read fault injection LID.", new object[0]);
return;
}
if (!this.ReadFaultInjectionParameters(ref line, ref parameters))
{
this.ReportError("Failed to read fault injection parameters.", new object[0]);
return;
}
lock (this.FaultInjectionConfig)
{
FaultInjectionTagComponentConfig faultInjectionTagComponentConfig = null;
FaultInjectionComponentConfig faultInjectionComponentConfig = null;
if (!this.FaultInjectionConfig.TryGetValue(traceComponentInfo.ComponentGuid, out faultInjectionTagComponentConfig))
{
faultInjectionTagComponentConfig = new FaultInjectionTagComponentConfig();
this.FaultInjectionConfig.Add(traceComponentInfo.ComponentGuid, faultInjectionTagComponentConfig);
}
if (!faultInjectionTagComponentConfig.TryGetValue(empty, out faultInjectionComponentConfig))
{
faultInjectionComponentConfig = new FaultInjectionComponentConfig();
faultInjectionTagComponentConfig.Add(empty, faultInjectionComponentConfig);
}
faultInjectionComponentConfig[key] = new FaultInjectionPointConfig(type, parameters);
}
}
public void Reset()
{
this.FaultInjectionType = FaultInjectionType.None;
this.FaultInjectionPhase = TransactionFaultInjectPhase.None;
}
public override Task MultiGrainWriteTransaction_FaultInjection(TransactionFaultInjectPhase injectionPhase, FaultInjectionType injectionType)
{
return(base.MultiGrainWriteTransaction_FaultInjection(injectionPhase, injectionType));
}
internal FaultInjectionPointConfig(FaultInjectionType type, List <string> parameters)
{
this.type = type;
this.parameters = parameters;
}