protected override IMemoizationStore CreateStore(DisposableDirectory testDirectory)
{
var context = new Context(_logger);
var keySpace = Guid.NewGuid().ToString();
var primaryRedisInstance = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock);
_databasesToDispose.Add(primaryRedisInstance);
var primaryFactory = RedisDatabaseFactory.CreateAsync(
context,
provider: new LiteralConnectionStringProvider(primaryRedisInstance.ConnectionString),
logSeverity: Severity.Info,
usePreventThreadTheft: false).GetAwaiter().GetResult();
var primaryRedisAdapter = new RedisDatabaseAdapter(primaryFactory, keySpace: keySpace);
var secondaryRedisInstance = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock);
_databasesToDispose.Add(secondaryRedisInstance);
var secondaryFactory = RedisDatabaseFactory.CreateAsync(
context,
provider: new LiteralConnectionStringProvider(secondaryRedisInstance.ConnectionString),
logSeverity: Severity.Info,
usePreventThreadTheft: false).GetAwaiter().GetResult();
var secondaryRedisAdapter = new RedisDatabaseAdapter(secondaryFactory, keySpace: keySpace);
var memoizationDb = new RedisMemoizationDatabase(primaryRedisAdapter, secondaryRedisAdapter, _clock, _memoizationExpiryTime, operationsTimeout: null, slowOperationRedisTimeout: null);
return(new RedisMemoizationStore(_logger, memoizationDb));
}
private async Task RunTest(Func <OperationContext, RedisWriteAheadEventStorage, Task> runTestAsync, RedisVolatileEventStorageConfiguration configuration = null)
{
var tracingContext = new Context(Logger);
var operationContext = new OperationContext(tracingContext);
using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance);
var primaryFactory = await RedisDatabaseFactory.CreateAsync(
operationContext,
new LiteralConnectionStringProvider(database.ConnectionString),
new RedisConnectionMultiplexerConfiguration()
{
LoggingSeverity = Severity.Error
});
var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace");
configuration ??= new RedisVolatileEventStorageConfiguration();
var instance = new RedisWriteAheadEventStorage(configuration, primaryDatabaseAdapter);
await instance.StartupAsync(operationContext).ThrowIfFailure();
await runTestAsync(operationContext, instance);
await instance.ShutdownAsync(operationContext).ThrowIfFailure();
}
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path / "Root";
var tempPath = testDirectory.Path / "Temp";
var configurationModel = new ConfigurationModel(configuration);
var fileCopier = new TestFileCopier();
var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var storeFactory = new MockRedisContentLocationStoreFactory(localDatabase, localMachineDatabase, rootPath);
return(new DistributedContentStore <AbsolutePath>(
storeFactory.LocalMachineData,
(nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance, rootPath,
configurationModel,
nagleQueue: nagleBlock,
distributedEvictionSettings: distributedEvictionSettings,
settings: contentStoreSettings,
trimBulkAsync: trimBulkAsync),
storeFactory,
fileCopier,
fileCopier,
storeFactory.PathTransformer,
ReadOnlyDistributedContentSession <AbsolutePath> .ContentAvailabilityGuarantee.FileRecordsExist,
tempPath,
FileSystem,
RedisContentLocationStoreConstants.DefaultBatchSize,
retryIntervalForCopies: DefaultRetryIntervalsForTest,
pinConfiguration: new PinConfiguration()));
}
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path / "Root";
var configurationModel = new ConfigurationModel(configuration);
var primaryDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var secondaryDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineLocation = new MachineLocation(rootPath.Path);
var storeFactory = new MockContentLocationStoreFactory(primaryDatabase, secondaryDatabase, rootPath);
var settings = CreateSettings();
return(new DistributedContentStore(
localMachineLocation,
rootPath,
(distributedStore) =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
rootPath,
configurationModel,
settings: ContentStoreSettings.DefaultSettings,
distributedStore: distributedStore),
storeFactory,
settings: settings,
distributedCopier: storeFactory.GetCopier()));
}
protected override ITestRedisDatabase GetRedisDatabase(
MemoryClock clock,
IDictionary <RedisKey, RedisValue> initialData = null,
IDictionary <RedisKey, DateTime> expiryData = null,
IDictionary <RedisKey, RedisValue[]> setData = null)
{
return(LocalRedisProcessDatabase.CreateAndStart(Redis, TestGlobal.Logger, clock, initialData, expiryData, setData));
}
public MockContentLocationStoreFactory(
LocalRedisProcessDatabase primaryRedisDatabase,
LocalRedisProcessDatabase secondaryRedisDatabase,
AbsolutePath rootDirectory,
ITestClock mockClock = null,
RedisContentLocationStoreConfiguration configuration = null)
: base(mockClock ?? TestSystemClock.Instance, configuration ?? CreateDefaultConfiguration(rootDirectory, primaryRedisDatabase, secondaryRedisDatabase), CreateTestCopier(mockClock ?? TestSystemClock.Instance, rootDirectory))
{
_primaryRedisDatabase = primaryRedisDatabase;
_secondaryRedisDatabase = secondaryRedisDatabase;
}
protected override IMemoizationStore CreateStore(DisposableDirectory testDirectory)
{
var context = new Context(_logger);
var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, _logger, _clock);
var connectionString = localDatabase.ConnectionString;
_databasesToDispose.Add(localDatabase);
var connectionStringProvider = new LiteralConnectionStringProvider(connectionString);
var redisFactory = RedisDatabaseFactory.CreateAsync(context, connectionStringProvider).GetAwaiter().GetResult();
var redisAdapter = new RedisDatabaseAdapter(redisFactory, keySpace: Guid.NewGuid().ToString());
var memoizationDb = new RedisMemoizationDatabase(redisAdapter, _clock, _memoizationExpiryTime);
return(new RedisMemoizationStore(_logger, memoizationDb));
}
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path / "Root";
var configurationModel = new ConfigurationModel(configuration);
var fileCopier = new TestFileCopier();
var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineLocation = new MachineLocation(rootPath.Path);
var storeFactory = new MockRedisContentLocationStoreFactory(localDatabase, localMachineDatabase, rootPath);
var settings = CreateSettings();
var distributedCopier = new DistributedContentCopier <AbsolutePath>(
settings,
FileSystem,
fileCopier,
fileCopier,
copyRequester: null,
storeFactory.PathTransformer,
SystemClock.Instance);
return(new DistributedContentStore <AbsolutePath>(
localMachineLocation,
rootPath,
(nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
rootPath,
configurationModel,
nagleQueue: nagleBlock,
distributedEvictionSettings: distributedEvictionSettings,
settings: contentStoreSettings,
trimBulkAsync: trimBulkAsync),
storeFactory,
settings: settings,
distributedCopier: distributedCopier));
}
private async Task RunTest(
Func <OperationContext, ContentMetadataEventStream, IFailureController, IFailureController, Task> runTestAsync,
ContentMetadataEventStreamConfiguration contentMetadataEventStreamConfiguration = null,
RedisVolatileEventStorageConfiguration redisVolatileEventLogConfiguration = null,
FailureMode persistentStorageFailure = FailureMode.None,
FailureMode volatileStorageFailure = FailureMode.None)
{
var tracingContext = new Context(Logger);
var operationContext = new OperationContext(tracingContext);
redisVolatileEventLogConfiguration ??= new RedisVolatileEventStorageConfiguration();
using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance);
var primaryFactory = await RedisDatabaseFactory.CreateAsync(
operationContext,
new LiteralConnectionStringProvider(database.ConnectionString),
new RedisConnectionMultiplexerConfiguration()
{
LoggingSeverity = Severity.Error
});
var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace");
var redisVolatileEventStorage = new RedisWriteAheadEventStorage(redisVolatileEventLogConfiguration, primaryDatabaseAdapter);
var mockPersistentEventStorage = new MockPersistentEventStorage();
var volatileEventStorage = new FailingVolatileEventStorage(volatileStorageFailure, redisVolatileEventStorage);
var persistentEventStorage = new FailingPersistentEventStorage(persistentStorageFailure, mockPersistentEventStorage);
contentMetadataEventStreamConfiguration ??= new ContentMetadataEventStreamConfiguration();
var contentMetadataEventStream = new ContentMetadataEventStream(contentMetadataEventStreamConfiguration, volatileEventStorage, persistentEventStorage);
await contentMetadataEventStream.StartupAsync(operationContext).ThrowIfFailure();
await contentMetadataEventStream.CompleteOrChangeLogAsync(operationContext, CheckpointLogId.InitialLogId);
contentMetadataEventStream.SetIsLogging(true);
await runTestAsync(operationContext, contentMetadataEventStream, volatileEventStorage, persistentEventStorage);
await contentMetadataEventStream.ShutdownAsync(operationContext).ThrowIfFailure();
}
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path / "Root";
var tempPath = testDirectory.Path / "Temp";
var configurationModel = new ConfigurationModel(configuration);
var fileCopier = new TestFileCopier();
var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineLocation = new MachineLocation(rootPath.Path);
var storeFactory = new MockRedisContentLocationStoreFactory(localDatabase, localMachineDatabase, rootPath);
return(new DistributedContentStore <AbsolutePath>(
localMachineLocation.Data,
(nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
rootPath,
configurationModel,
nagleQueue: nagleBlock,
distributedEvictionSettings: distributedEvictionSettings,
settings: contentStoreSettings,
trimBulkAsync: trimBulkAsync),
storeFactory,
fileCopier,
fileCopier,
storeFactory.PathTransformer,
copyRequester: null,
tempPath,
FileSystem,
settings: new DistributedContentStoreSettings
{
ContentAvailabilityGuarantee = ContentAvailabilityGuarantee.FileRecordsExist,
LocationStoreBatchSize = RedisContentLocationStoreConstants.DefaultBatchSize,
RetryIntervalForCopies = DefaultRetryIntervalsForTest,
SetPostInitializationCompletionAfterStartup = true
}));
}
private static RedisContentLocationStoreConfiguration CreateDefaultConfiguration(
AbsolutePath rootDirectory,
LocalRedisProcessDatabase primaryRedisDatabase,
LocalRedisProcessDatabase secondaryRedisDatabase)
{
var configuration = new RedisContentLocationStoreConfiguration()
{
PrimaryMachineLocation = new MachineLocation(rootDirectory.ToString()),
Keyspace = "Default:",
RedisGlobalStoreConnectionString = primaryRedisDatabase.ConnectionString,
RedisGlobalStoreSecondaryConnectionString = secondaryRedisDatabase?.ConnectionString
};
configuration.InlinePostInitialization = true;
configuration.MachineStateRecomputeInterval = TimeSpan.Zero;
configuration.EventStore = new MemoryContentLocationEventStoreConfiguration();
configuration.Database = new RocksDbContentLocationDatabaseConfiguration(rootDirectory / "rocksdb");
configuration.Checkpoint = new CheckpointConfiguration(rootDirectory, configuration.PrimaryMachineLocation);
configuration.CentralStore = new LocalDiskCentralStoreConfiguration(rootDirectory / "centralStore", "checkpoints-key");
return(configuration);
}
protected override IContentStore CreateStore(
Context context,
TestFileCopier fileCopier,
DisposableDirectory testDirectory,
int index,
bool enableDistributedEviction,
int?replicaCreditInMinutes,
bool enableRepairHandling,
object additionalArgs)
{
var rootPath = testDirectory.Path / "Root";
var tempPath = testDirectory.Path / "Temp";
var configurationModel = new ConfigurationModel(Config);
var pathTransformer = new TestPathTransformer();
var localMachineData = pathTransformer.GetLocalMachineLocation(rootPath);
if (!_localDatabases.TryGetValue(context.Id, out var localDatabase))
{
localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
_localDatabases.TryAdd(context.Id, localDatabase);
}
if (!_localMachineDatabases.TryGetValue(context.Id, out var localMachineDatabase))
{
localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
_localMachineDatabases.TryAdd(context.Id, localMachineDatabase);
}
if (enableDistributedEviction && replicaCreditInMinutes == null)
{
// Apparently, replicaCreditInMinutes != null enables distributed eviction,
// so make sure replicaCreditInMinutes is set when enableDistributedEviction is
// true
replicaCreditInMinutes = 0;
}
_configuration = CreateContentLocationStoreConfiguration?.Invoke(rootPath, index) ?? new RedisContentLocationStoreConfiguration();
_configuration.BlobExpiryTimeMinutes = 10;
PostProcessConfiguration(_configuration, index);
var storeFactory = new MockRedisContentLocationStoreFactory(
localDatabase,
localMachineDatabase,
rootPath,
mockClock: TestClock,
_configuration);
var distributedContentStore = new DistributedContentStore <AbsolutePath>(
localMachineData,
(nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) =>
new FileSystemContentStore(
FileSystem,
TestClock,
rootPath,
configurationModel,
nagleQueue: nagleBlock,
distributedEvictionSettings: distributedEvictionSettings,
settings: contentStoreSettings,
trimBulkAsync: trimBulkAsync),
storeFactory,
fileCopier,
fileCopier,
pathTransformer,
fileCopier,
ContentAvailabilityGuarantee,
tempPath,
FileSystem,
RedisContentLocationStoreConstants.DefaultBatchSize,
settings: new DistributedContentStoreSettings
{
RetryIntervalForCopies = DistributedContentSessionTests.DefaultRetryIntervalsForTest,
PinConfiguration = PinConfiguration,
ShouldInlinePutBlob = true,
},
replicaCreditInMinutes: replicaCreditInMinutes,
clock: TestClock,
enableRepairHandling: enableRepairHandling,
contentStoreSettings: new ContentStoreSettings()
{
CheckFiles = true,
},
setPostInitializationCompletionAfterStartup: true);
distributedContentStore.DisposeContentStoreFactory = false;
return(distributedContentStore);
}
private async Task RunTest(
Func <OperationContext, ResilientGlobalCacheService, int, Task> runTestAsync,
bool persistentStorageFailure = false,
bool volatileStorageFailure = false,
IClock?clock = null,
int iterations = 1,
Action <GlobalCacheServiceConfiguration>?modifyConfig = null)
{
var tracingContext = new Context(Logger);
var operationContext = new OperationContext(tracingContext);
clock ??= SystemClock.Instance;
var contentMetadataServiceConfiguration = new GlobalCacheServiceConfiguration()
{
Checkpoint = new CheckpointManagerConfiguration(TestRootDirectoryPath / "CheckpointManager", default(MachineLocation)),
EventStream = new ContentMetadataEventStreamConfiguration(),
};
modifyConfig?.Invoke(contentMetadataServiceConfiguration);
using var database = LocalRedisProcessDatabase.CreateAndStartEmpty(_redisFixture, TestGlobal.Logger, SystemClock.Instance);
var primaryFactory = await RedisDatabaseFactory.CreateAsync(
operationContext,
new LiteralConnectionStringProvider(database.ConnectionString),
new RedisConnectionMultiplexerConfiguration()
{
LoggingSeverity = Severity.Error
});
var primaryDatabaseAdapter = new RedisDatabaseAdapter(primaryFactory, "keyspace");
var centralStorage = new Dictionary <string, byte[]>();
for (var iteration = 0; iteration < iterations; iteration++)
{
Tracer.Info(operationContext, $"Running iteration {iteration}");
var redisVolatileEventStorage = new RedisWriteAheadEventStorage(new RedisVolatileEventStorageConfiguration(), primaryDatabaseAdapter, clock);
IWriteAheadEventStorage volatileEventStorage = new FailingVolatileEventStorage();
if (!volatileStorageFailure)
{
volatileEventStorage = redisVolatileEventStorage;
}
IWriteBehindEventStorage persistentEventStorage = new FailingPersistentEventStorage();
if (!persistentStorageFailure)
{
persistentEventStorage = new MockPersistentEventStorage();
}
var contentMetadataEventStream = new ContentMetadataEventStream(
contentMetadataServiceConfiguration.EventStream,
volatileEventStorage,
persistentEventStorage);
var rocksdbContentMetadataDatabaseConfiguration = new RocksDbContentMetadataDatabaseConfiguration(TestRootDirectoryPath / "ContentMetadataDatabase");
var rocksDbContentMetadataStore = new RocksDbContentMetadataStore(clock, new RocksDbContentMetadataStoreConfiguration()
{
Database = rocksdbContentMetadataDatabaseConfiguration,
});
var storage = new MockCentralStorage(centralStorage);
var checkpointManager = new CheckpointManager(
rocksDbContentMetadataStore.Database,
redisVolatileEventStorage,
storage,
contentMetadataServiceConfiguration.Checkpoint,
new CounterCollection <ContentLocationStoreCounters>());
var resilientContentMetadataService = new ResilientGlobalCacheService(
contentMetadataServiceConfiguration,
checkpointManager,
rocksDbContentMetadataStore,
contentMetadataEventStream,
storage,
clock);
await resilientContentMetadataService.StartupAsync(operationContext).ThrowIfFailure();
await runTestAsync(operationContext, resilientContentMetadataService, iteration);
await resilientContentMetadataService.ShutdownAsync(operationContext).ThrowIfFailure();
}
}
