public ConsistencyTestHarness(
IGrainFactory grainFactory,
int numGrains,
int seed,
bool avoidDeadlocks,
bool avoidTimeouts,
ReadWriteDetermination readWrite,
bool tolerateUnknownExceptions)
{
this.grainFactory = grainFactory;
numGrains.Should().BeLessThan(ConsistencyTestOptions.MaxGrains);
this.options = new ConsistencyTestOptions()
{
AvoidDeadlocks = avoidDeadlocks,
ReadWrite = readWrite,
MaxDepth = 5,
NumGrains = numGrains,
RandomSeed = seed,
AvoidTimeouts = avoidTimeouts,
GrainOffset = (DateTime.UtcNow.Ticks & 0xFFFFFFFF) * ConsistencyTestOptions.MaxGrains,
};
this.tuples = new Dictionary <int, SortedDictionary <int, Dictionary <string, HashSet <string> > > >();
this.succeeded = new HashSet <string>();
this.aborted = new HashSet <string>();
this.indoubt = new Dictionary <string, string>();
// determine what to check for in the end
this.tolerateUnknownExceptions = tolerateUnknownExceptions;
}
public virtual async Task RandomizedConsistency(int numGrains, int scale, bool avoidDeadlocks, bool avoidTimeouts, ReadWriteDetermination readwrite)
{
var random = new Random(scale + numGrains * 1000 + (avoidDeadlocks ? 666 : 333) + ((int)readwrite) * 123976);
var harness = new ConsistencyTestHarness(grainFactory, numGrains, random.Next(), avoidDeadlocks, avoidTimeouts, readwrite, StorageErrorInjectionActive);
// first, run the random work load to generate history events
testOutput($"start at {DateTime.UtcNow}");
int numThreads = scale;
int numTxsPerThread = scale * scale;
// start the threads that run transactions
var tasks = new Task[numThreads];
for (int i = 0; i < numThreads; i++)
{
tasks[i] = harness.RunRandomTransactionSequence(i, numTxsPerThread, grainFactory, this.testOutput);
}
// wait for the test to finish
await Task.WhenAll(tasks);
testOutput($"end at {DateTime.UtcNow}");
// golden path: all transactions are expected to pass when avoiding deadlocks and lock upgrades
if (!StorageErrorInjectionActive &&
avoidDeadlocks &&
(readwrite == ReadWriteDetermination.PerGrain || readwrite == ReadWriteDetermination.PerTransaction))
{
harness.NumAborted.Should().Be(0);
}
// then, analyze the history results
var tolerateGenericTimeouts = StorageErrorInjectionActive || (scale >= 3 && !avoidTimeouts);
var tolerateUnknownExceptions = StorageAdaptorHasLimitedCommitSpace || StorageErrorInjectionActive;
harness.CheckConsistency(tolerateGenericTimeouts, tolerateUnknownExceptions);
}
public override Task RandomizedConsistency(int numGrains, int scale, bool avoidDeadlocks,
bool avoidTimeouts, ReadWriteDetermination readwrite)
{
return(base.RandomizedConsistency(numGrains, scale, avoidDeadlocks, avoidTimeouts, readwrite));
}