private async Task RunTest(
Func <OperationContext, ContentMetadataEventStream, IFailureController, IFailureController, Task> runTestAsync,
ContentMetadataEventStreamConfiguration contentMetadataEventStreamConfiguration = null,
RedisVolatileEventStorageConfiguration redisVolatileEventLogConfiguration = null,
FailureMode persistentStorageFailure = FailureMode.None,
FailureMode volatileStorageFailure = FailureMode.None)
{
var tracingContext = new Context(Logger);
var operationContext = new OperationContext(tracingContext);
redisVolatileEventLogConfiguration ??= new RedisVolatileEventStorageConfiguration();
using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance);
var primaryFactory = await RedisDatabaseFactory.CreateAsync(
operationContext,
new LiteralConnectionStringProvider(database.ConnectionString),
new RedisConnectionMultiplexerConfiguration()
{
LoggingSeverity = Severity.Error
});
var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace");
var redisVolatileEventStorage = new RedisWriteAheadEventStorage(redisVolatileEventLogConfiguration, primaryDatabaseAdapter);
var mockPersistentEventStorage = new MockPersistentEventStorage();
var volatileEventStorage = new FailingVolatileEventStorage(volatileStorageFailure, redisVolatileEventStorage);
var persistentEventStorage = new FailingPersistentEventStorage(persistentStorageFailure, mockPersistentEventStorage);
contentMetadataEventStreamConfiguration ??= new ContentMetadataEventStreamConfiguration();
var contentMetadataEventStream = new ContentMetadataEventStream(contentMetadataEventStreamConfiguration, volatileEventStorage, persistentEventStorage);
await contentMetadataEventStream.StartupAsync(operationContext).ThrowIfFailure();
await contentMetadataEventStream.CompleteOrChangeLogAsync(operationContext, CheckpointLogId.InitialLogId);
contentMetadataEventStream.SetIsLogging(true);
await runTestAsync(operationContext, contentMetadataEventStream, volatileEventStorage, persistentEventStorage);
await contentMetadataEventStream.ShutdownAsync(operationContext).ThrowIfFailure();
}
private Task <Result <List <ServiceRequestBase> > > CollectStream(OperationContext context, ContentMetadataEventStream stream, CheckpointLogId cursor)
{
var msg = $"Cursor=[{cursor}]";
return(context.PerformOperationAsync(Tracer, async() =>
{
var events = new List <ServiceRequestBase>();
await stream.ReadEventsAsync(context, cursor, r =>
{
events.Add(r);
return new ValueTask(Task.CompletedTask);
}).ThrowIfFailureAsync();
return Result.Success(events);
}, extraStartMessage: msg, extraEndMessage: _ => msg));
}
private async Task RunTest(
Func <OperationContext, ResilientGlobalCacheService, int, Task> runTestAsync,
bool persistentStorageFailure = false,
bool volatileStorageFailure = false,
IClock?clock = null,
int iterations = 1,
Action <GlobalCacheServiceConfiguration>?modifyConfig = null)
{
var tracingContext = new Context(Logger);
var operationContext = new OperationContext(tracingContext);
clock ??= SystemClock.Instance;
var contentMetadataServiceConfiguration = new GlobalCacheServiceConfiguration()
{
Checkpoint = new CheckpointManagerConfiguration(TestRootDirectoryPath / "CheckpointManager", default(MachineLocation)),
EventStream = new ContentMetadataEventStreamConfiguration(),
};
modifyConfig?.Invoke(contentMetadataServiceConfiguration);
using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance);
var primaryFactory = await RedisDatabaseFactory.CreateAsync(
operationContext,
new LiteralConnectionStringProvider(database.ConnectionString),
new RedisConnectionMultiplexerConfiguration()
{
LoggingSeverity = Severity.Error
});
var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace");
var centralStorage = new Dictionary <string, byte[]>();
for (var iteration = 0; iteration < iterations; iteration++)
{
Tracer.Info(operationContext, $"Running iteration {iteration}");
var redisVolatileEventStorage = new RedisWriteAheadEventStorage(new RedisVolatileEventStorageConfiguration(), primaryDatabaseAdapter, clock);
IWriteAheadEventStorage volatileEventStorage = new FailingVolatileEventStorage();
if (!volatileStorageFailure)
{
volatileEventStorage = redisVolatileEventStorage;
}
IWriteBehindEventStorage persistentEventStorage = new FailingPersistentEventStorage();
if (!persistentStorageFailure)
{
persistentEventStorage = new MockPersistentEventStorage();
}
var contentMetadataEventStream = new ContentMetadataEventStream(
contentMetadataServiceConfiguration.EventStream,
volatileEventStorage,
persistentEventStorage);
var rocksdbContentMetadataDatabaseConfiguration = new RocksDbContentMetadataDatabaseConfiguration(TestRootDirectoryPath / "ContentMetadataDatabase");
var rocksDbContentMetadataStore = new RocksDbContentMetadataStore(clock, new RocksDbContentMetadataStoreConfiguration()
{
Database = rocksdbContentMetadataDatabaseConfiguration,
});
var storage = new MockCentralStorage(centralStorage);
var checkpointManager = new CheckpointManager(
rocksDbContentMetadataStore.Database,
redisVolatileEventStorage,
storage,
contentMetadataServiceConfiguration.Checkpoint,
new CounterCollection <ContentLocationStoreCounters>());
var resilientContentMetadataService = new ResilientGlobalCacheService(
contentMetadataServiceConfiguration,
checkpointManager,
rocksDbContentMetadataStore,
contentMetadataEventStream,
storage,
clock);
await resilientContentMetadataService.StartupAsync(operationContext).ThrowIfFailure();
await runTestAsync(operationContext, resilientContentMetadataService, iteration);
await resilientContentMetadataService.ShutdownAsync(operationContext).ThrowIfFailure();
}
}
