protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path / "Root";
var tempPath = testDirectory.Path / "Temp";
var configurationModel = new ConfigurationModel(configuration);
var fileCopier = new TestFileCopier();
var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineLocation = new MachineLocation(rootPath.Path);
var storeFactory = new MockRedisContentLocationStoreFactory(localDatabase, localMachineDatabase, rootPath);
return(new DistributedContentStore <AbsolutePath>(
localMachineLocation.Data,
(nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
rootPath,
configurationModel,
nagleQueue: nagleBlock,
distributedEvictionSettings: distributedEvictionSettings,
settings: contentStoreSettings,
trimBulkAsync: trimBulkAsync),
storeFactory,
fileCopier,
fileCopier,
storeFactory.PathTransformer,
copyRequester: null,
tempPath,
FileSystem,
settings: new DistributedContentStoreSettings
{
ContentAvailabilityGuarantee = ContentAvailabilityGuarantee.FileRecordsExist,
LocationStoreBatchSize = RedisContentLocationStoreConstants.DefaultBatchSize,
RetryIntervalForCopies = DefaultRetryIntervalsForTest,
SetPostInitializationCompletionAfterStartup = true
}));
}
private Task <(TestDistributedContentCopier, MockFileCopier)> CreateAsync(Context context, AbsolutePath rootDirectory, TimeSpan retryInterval, int retries = 1, MachineLocation[] designatedLocations = null)
{
var mockFileCopier = new MockFileCopier();
var existenceChecker = new TestFileCopier();
var contentCopier = new TestDistributedContentCopier(
rootDirectory,
// Need to use exactly one retry.
new DistributedContentStoreSettings()
{
RetryIntervalForCopies = Enumerable.Range(0, retries).Select(r => retryInterval).ToArray(),
PrioritizeDesignatedLocationsOnCopies = designatedLocations != null,
TrustedHashFileSizeBoundary = long.MaxValue // Disable trusted hash because we never actually move bytes and thus the hasher thinks there is a mismatch.
},
FileSystem,
mockFileCopier,
existenceChecker,
copyRequester: null,
new NoOpPathTransformer(rootDirectory),
designatedLocations);
return(Task.FromResult((contentCopier, mockFileCopier)));
}
public TestContext(
DistributedContentTests <TStore, TSession> testInstance,
Context context,
IRemoteFileCopier fileCopier,
IList <DisposableDirectory> directories,
IReadOnlyList <TestServerProvider> serverProviders,
int iteration,
int[] ports,
bool traceStoreStatistics = false)
{
_testInstance = testInstance;
_traceStoreStatistics = traceStoreStatistics;
Context = context;
StoreContexts = serverProviders.Select((s, index) => new OperationContext(CreateContext(index, iteration))).ToArray();
TestFileCopier = fileCopier as TestFileCopier;
FileCopier = fileCopier;
Directories = directories;
ServerProviders = serverProviders;
Stores = serverProviders.SelectList(s => s.Store);
Servers = serverProviders.Select(s => s.Server).ToList();
Iteration = iteration;
Ports = ports;
}
public static (TestDistributedContentCopier, MockFileCopier) CreateMocks(
IAbsFileSystem fileSystem,
AbsolutePath rootDirectory,
TimeSpan retryInterval,
int retries = 1)
{
var mockFileCopier = new MockFileCopier();
var existenceChecker = new TestFileCopier();
var contentCopier = new TestDistributedContentCopier(
rootDirectory,
// Need to use exactly one retry.
new DistributedContentStoreSettings()
{
RetryIntervalForCopies = Enumerable.Range(0, retries).Select(r => retryInterval).ToArray(),
TrustedHashFileSizeBoundary = long.MaxValue // Disable trusted hash because we never actually move bytes and thus the hasher thinks there is a mismatch.
},
fileSystem,
mockFileCopier,
existenceChecker,
copyRequester: null,
new TestDistributedContentCopier.NoOpPathTransformer(rootDirectory));
return(contentCopier, mockFileCopier);
}
protected async Task RunTestAsync(
Context context,
int storeCount,
Func <TestContext, Task> testFunc,
ImplicitPin implicitPin = ImplicitPin.PutAndGet,
bool enableDistributedEviction = false,
int?replicaCreditInMinutes = null,
bool enableRepairHandling = false,
bool emptyFileHashShortcutEnabled = false,
int iterations = 1)
{
var indexedDirectories = Enumerable.Range(0, storeCount)
.Select(i => new { Index = i, Directory = new DisposableDirectory(FileSystem, TestRootDirectoryPath / i.ToString()) })
.ToList();
var testFileCopier = new TestFileCopier();
for (int iteration = 0; iteration < iterations; iteration++)
{
var stores = indexedDirectories.Select(
directory =>
CreateStore(
context,
testFileCopier,
directory.Directory,
directory.Index,
enableDistributedEviction,
replicaCreditInMinutes,
enableRepairHandling,
emptyFileHashShortcutEnabled)).ToList();
var startupResults = await TaskSafetyHelpers.WhenAll(stores.Select(async store => await store.StartupAsync(context)));
Assert.True(startupResults.All(x => x.Succeeded), $"Failed to startup: {string.Join(Environment.NewLine, startupResults.Where(s => !s))}");
var id = 0;
var sessions = stores.Select(store => store.CreateSession(context, "store" + id++, implicitPin).Session).ToList();
await TaskSafetyHelpers.WhenAll(sessions.Select(async session => await session.StartupAsync(context)));
var testContext = new TestContext(context, testFileCopier, indexedDirectories.Select(p => p.Directory).ToList(), sessions, stores, iteration);
await testFunc(testContext);
await TaskSafetyHelpers.WhenAll(
sessions.Select(async session =>
{
if (!session.ShutdownCompleted)
{
await session.ShutdownAsync(context).ThrowIfFailure();
}
}));
sessions.ForEach(session => session.Dispose());
await TaskSafetyHelpers.WhenAll(Enumerable.Range(0, storeCount).Select(storeId => LogStats(testContext, storeId)));
await TaskSafetyHelpers.WhenAll(stores.Select(async store => await store.ShutdownAsync(context)));
stores.ForEach(store => store.Dispose());
}
indexedDirectories.ForEach(directory => directory.Directory.Dispose());
}
protected override IContentStore CreateStore(
Context context,
TestFileCopier fileCopier,
DisposableDirectory testDirectory,
int index,
bool enableDistributedEviction,
int?replicaCreditInMinutes,
bool enableRepairHandling,
object additionalArgs)
{
var rootPath = testDirectory.Path / "Root";
var tempPath = testDirectory.Path / "Temp";
var configurationModel = new ConfigurationModel(Config);
var pathTransformer = new TestPathTransformer();
var localMachineData = pathTransformer.GetLocalMachineLocation(rootPath);
if (!_localDatabases.TryGetValue(context.Id, out var localDatabase))
{
localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
_localDatabases.TryAdd(context.Id, localDatabase);
}
if (!_localMachineDatabases.TryGetValue(context.Id, out var localMachineDatabase))
{
localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
_localMachineDatabases.TryAdd(context.Id, localMachineDatabase);
}
if (enableDistributedEviction && replicaCreditInMinutes == null)
{
// Apparently, replicaCreditInMinutes != null enables distributed eviction,
// so make sure replicaCreditInMinutes is set when enableDistributedEviction is
// true
replicaCreditInMinutes = 0;
}
_configuration = CreateContentLocationStoreConfiguration?.Invoke(rootPath, index) ?? new RedisContentLocationStoreConfiguration();
_configuration.BlobExpiryTimeMinutes = 10;
PostProcessConfiguration(_configuration, index);
var storeFactory = new MockRedisContentLocationStoreFactory(
localDatabase,
localMachineDatabase,
rootPath,
mockClock: TestClock,
_configuration);
var distributedContentStore = new DistributedContentStore <AbsolutePath>(
localMachineData,
(nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) =>
new FileSystemContentStore(
FileSystem,
TestClock,
rootPath,
configurationModel,
nagleQueue: nagleBlock,
distributedEvictionSettings: distributedEvictionSettings,
settings: contentStoreSettings,
trimBulkAsync: trimBulkAsync),
storeFactory,
fileCopier,
fileCopier,
pathTransformer,
fileCopier,
ContentAvailabilityGuarantee,
tempPath,
FileSystem,
RedisContentLocationStoreConstants.DefaultBatchSize,
settings: new DistributedContentStoreSettings
{
RetryIntervalForCopies = DistributedContentSessionTests.DefaultRetryIntervalsForTest,
PinConfiguration = PinConfiguration,
ShouldInlinePutBlob = true,
},
replicaCreditInMinutes: replicaCreditInMinutes,
clock: TestClock,
enableRepairHandling: enableRepairHandling,
contentStoreSettings: new ContentStoreSettings()
{
CheckFiles = true,
},
setPostInitializationCompletionAfterStartup: true);
distributedContentStore.DisposeContentStoreFactory = false;
return(distributedContentStore);
}
