internal DistributedCacheServiceArguments CreateDistributedCacheServiceArguments(
IAbsolutePathFileCopier copier,
IAbsolutePathTransformer pathTransformer,
ICopyRequester copyRequester,
DistributedContentSettings dcs,
HostInfo host,
string cacheName,
string cacheRootPath,
uint grpcPort,
int maxSizeQuotaMB,
string dataRootPath,
CancellationToken ct,
int?bufferSizeForGrpcCopies,
int?gzipBarrierSizeForGrpcCopies)
{
var distributedCacheServiceHost = new EnvironmentVariableHost();
var localCasSettings = LocalCasSettings.Default(
maxSizeQuotaMB: maxSizeQuotaMB,
cacheRootPath: cacheRootPath,
cacheName: cacheName,
grpcPort: grpcPort,
grpcPortFileName: _scenario);
localCasSettings.PreferredCacheDrive = Path.GetPathRoot(cacheRootPath);
localCasSettings.ServiceSettings = new LocalCasServiceSettings(60, scenarioName: _scenario, grpcPort: grpcPort, grpcPortFileName: _scenario, bufferSizeForGrpcCopies: bufferSizeForGrpcCopies, gzipBarrierSizeForGrpcCopies: gzipBarrierSizeForGrpcCopies,
grpcThreadPoolSize: null);
var config = new DistributedCacheServiceConfiguration(localCasSettings, dcs);
return(new DistributedCacheServiceArguments(_logger, copier, pathTransformer, copyRequester, distributedCacheServiceHost, host, ct, dataRootPath, config, null));
}
/// <nodoc />
public DistributedContentCopier(
AbsolutePath workingDirectory,
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IFileCopier <T> fileCopier,
IFileExistenceChecker <T> fileExistenceChecker,
ICopyRequester copyRequester,
IPathTransformer <T> pathTransformer,
IContentLocationStore contentLocationStore)
{
Contract.Requires(settings != null);
Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);
_settings = settings;
_tempFolderForCopies = new DisposableDirectory(fileSystem, workingDirectory / "Temp");
_remoteFileCopier = fileCopier;
_remoteFileExistenceChecker = fileExistenceChecker;
_copyRequester = copyRequester;
_contentLocationStore = contentLocationStore;
_pathTransformer = pathTransformer;
_fileSystem = fileSystem;
_workingDirectory = _tempFolderForCopies.Path;
// TODO: Use hashers from IContentStoreInternal instead?
_hashers = HashInfoLookup.CreateAll();
_ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations);
_retryIntervals = settings.RetryIntervalForCopies;
}
/// <nodoc />
public DistributedContentCopier(
AbsolutePath workingDirectory,
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IFileCopier <T> fileCopier,
IFileExistenceChecker <T> fileExistenceChecker,
ICopyRequester copyRequester,
IPathTransformer <T> pathTransformer,
IContentLocationStore contentLocationStore)
{
Contract.Requires(settings != null);
Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);
_settings = settings;
_tempFolderForCopies = new DisposableDirectory(fileSystem, workingDirectory / "Temp");
_remoteFileCopier = fileCopier;
_remoteFileExistenceChecker = fileExistenceChecker;
_copyRequester = copyRequester;
_contentLocationStore = contentLocationStore;
_pathTransformer = pathTransformer;
_fileSystem = fileSystem;
_workingDirectory = _tempFolderForCopies.Path;
_ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations);
_proactiveCopyIoGate = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations);
_retryIntervals = settings.RetryIntervalForCopies;
_maxRetryCount = settings.MaxRetryCount;
_timeoutForPoractiveCopies = settings.TimeoutForProactiveCopies;
}
/// <nodoc />
public DistributedContentStore(
byte[] localMachineLocation,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
IFileExistenceChecker <T> fileExistenceChecker,
IFileCopier <T> fileCopier,
IPathTransformer <T> pathTransform,
ICopyRequester copyRequester,
ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee,
AbsolutePath tempFolderForCopies,
IAbsFileSystem fileSystem,
int locationStoreBatchSize,
IReadOnlyList <TimeSpan> retryIntervalForCopies = null,
PinConfiguration pinConfiguration = null,
int?replicaCreditInMinutes = null,
IClock clock = null,
bool enableRepairHandling = false,
TimeSpan?contentHashBumpTime = null,
bool useTrustedHash = false,
int trustedHashFileSizeBoundary = -1,
long parallelHashingFileSizeBoundary = -1,
int maxConcurrentCopyOperations = 512,
ContentStoreSettings contentStoreSettings = null,
bool enableProactiveCopy = false)
: this(
localMachineLocation,
innerContentStoreFunc,
contentLocationStoreFactory,
fileExistenceChecker,
fileCopier,
pathTransform,
copyRequester,
contentAvailabilityGuarantee,
tempFolderForCopies,
fileSystem,
locationStoreBatchSize,
new DistributedContentStoreSettings()
{
UseTrustedHash = useTrustedHash,
TrustedHashFileSizeBoundary = trustedHashFileSizeBoundary,
ParallelHashingFileSizeBoundary = parallelHashingFileSizeBoundary,
MaxConcurrentCopyOperations = maxConcurrentCopyOperations,
RetryIntervalForCopies = retryIntervalForCopies,
PinConfiguration = pinConfiguration,
EnableProactiveCopy = enableProactiveCopy
},
replicaCreditInMinutes,
clock,
enableRepairHandling,
contentHashBumpTime,
contentStoreSettings)
{
// This constructor is used from tests,
// so we need to complete _postInitializationCompletion when startup is done.
_setPostInitializationCompletionAfterStartup = true;
}
protected abstract IContentStore CreateStore(
Context context,
TestFileCopier fileCopier,
ICopyRequester copyRequester,
DisposableDirectory testDirectory,
int index,
bool enableDistributedEviction,
int?replicaCreditInMinutes,
bool enableRepairHandling,
bool emptyFileHashShortcutEnabled,
object additionalArgs);
public TestDistributedContentCopier(
AbsolutePath workingDirectory,
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IFileCopier <AbsolutePath> fileCopier,
IFileExistenceChecker <AbsolutePath> fileExistenceChecker,
ICopyRequester copyRequester,
IPathTransformer <AbsolutePath> pathTransformer,
IContentLocationStore contentLocationStore)
: base(workingDirectory, settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransformer, contentLocationStore)
{
}
protected override IContentStore CreateStore(
Context context,
TestFileCopier fileCopier,
ICopyRequester copyRequester,
DisposableDirectory testDirectory,
int index,
bool enableDistributedEviction,
int?replicaCreditInMinutes,
bool enableRepairHandling,
bool emptyFileHashShortcutEnabled,
object additionalArgs)
{
if (index == 0)
{
return(base.CreateStore(context, fileCopier, copyRequester, testDirectory, index, enableDistributedEviction, replicaCreditInMinutes, enableRepairHandling, emptyFileHashShortcutEnabled, additionalArgs));
}
var grpcPortsByStoreIndex = additionalArgs as int[];
var configuration = new ContentStoreConfiguration();
var rootPath = testDirectory.Path;
configuration.Write(FileSystem, rootPath).Wait();
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfiguration = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ "Default", rootPath }
},
rootPath,
ServiceConfiguration.DefaultMaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPortsByStoreIndex[index],
grpcPortFileName);
// HACK: This will only work for 3 machines. We know that master is machine 0, and workes are 1 and 2.
copyRequester = index == 0 || grpcPortsByStoreIndex.Length < 3
? copyRequester // Master does not need a copy requester.
: new GrpcFileCopier(context, grpcPortsByStoreIndex[index == 1 ? 2 : 1], 16, 1, 1); // Point workers towards each other.
return(new TestInProcessServiceClientContentStore(
FileSystem,
Logger,
"Default",
grpcPortFileName,
null,
serviceConfiguration,
// Ignore path since we configured only one cache in the testDirectory root.
contentStoreFactory: _ => new SessionCapturingStore(base.CreateStore(context, fileCopier, copyRequester, testDirectory, index, enableDistributedEviction, replicaCreditInMinutes, enableRepairHandling, emptyFileHashShortcutEnabled, additionalArgs))
));
}
/// <inheritdoc />
public DistributedCacheServiceArguments(
ILogger logger,
IAbsolutePathFileCopier copier,
IAbsolutePathTransformer pathTransformer,
ICopyRequester copyRequester,
IDistributedCacheServiceHost host,
HostInfo hostInfo,
CancellationToken cancellation,
string dataRootPath,
DistributedCacheServiceConfiguration configuration,
string keyspace)
{
Logger = logger;
Copier = copier;
CopyRequester = copyRequester;
PathTransformer = pathTransformer;
Host = host;
HostInfo = hostInfo;
Cancellation = cancellation;
DataRootPath = dataRootPath;
Configuration = configuration;
Keyspace = ComputeKeySpace(hostInfo, configuration, keyspace);
}
/// <nodoc />
public DistributedContentStore(
byte[] localMachineLocation,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
IFileExistenceChecker <T> fileExistenceChecker,
IFileCopier <T> fileCopier,
IPathTransformer <T> pathTransform,
ICopyRequester copyRequester,
ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee,
AbsolutePath tempFolderForCopies,
IAbsFileSystem fileSystem,
int locationStoreBatchSize,
DistributedContentStoreSettings settings,
int?replicaCreditInMinutes = null,
IClock clock = null,
bool enableRepairHandling = false,
TimeSpan?contentHashBumpTime = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = new MachineLocation(localMachineLocation);
_enableRepairHandling = enableRepairHandling;
_contentLocationStoreFactory = contentLocationStoreFactory;
_contentAvailabilityGuarantee = contentAvailabilityGuarantee;
_locationStoreBatchSize = locationStoreBatchSize;
contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
_settings = settings;
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup.
_evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
_locationStoreBatchSize);
_distributedCopierFactory = (contentLocationStore) =>
{
return(new DistributedContentCopier <T>(
tempFolderForCopies,
_settings,
fileSystem,
fileCopier,
fileExistenceChecker,
copyRequester,
pathTransform,
contentLocationStore));
};
_enableDistributedEviction = replicaCreditInMinutes != null;
var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(replicaCreditInMinutes, locationStoreBatchSize) : null;
var enableTouch = contentHashBumpTime.HasValue;
if (enableTouch)
{
_contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, contentHashBumpTime.Value, clock: clock);
}
TrimBulkAsync trimBulkAsync = null;
InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);
if (settings.PinConfiguration?.UsePinCache == true)
{
_pinCache = new PinCache(clock: clock);
}
}
protected override IContentStore CreateStore(
Context context,
TestFileCopier fileCopier,
ICopyRequester copyRequester,
DisposableDirectory testDirectory,
int index,
bool enableDistributedEviction,
int?replicaCreditInMinutes,
bool enableRepairHandling,
bool emptyFileHashShortcutEnabled,
object additionalArgs)
{
var rootPath = testDirectory.Path / "Root";
var tempPath = testDirectory.Path / "Temp";
var configurationModel = new ConfigurationModel(Config);
var pathTransformer = new TestPathTransformer();
var localMachineData = pathTransformer.GetLocalMachineLocation(rootPath);
if (!_localDatabases.TryGetValue(context.Id, out var localDatabase))
{
localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
_localDatabases.TryAdd(context.Id, localDatabase);
}
if (!_localMachineDatabases.TryGetValue(context.Id, out var localMachineDatabase))
{
localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
_localMachineDatabases.TryAdd(context.Id, localMachineDatabase);
}
if (enableDistributedEviction && replicaCreditInMinutes == null)
{
// Apparently, replicaCreditInMinutes != null enables distributed eviction,
// so make sure replicaCreditInMinutes is set when enableDistributedEviction is
// true
replicaCreditInMinutes = 0;
}
_configuration = CreateContentLocationStoreConfiguration?.Invoke(rootPath, index) ?? new RedisContentLocationStoreConfiguration();
_configuration.BlobExpiryTimeMinutes = 10;
PostProcessConfiguration(_configuration, index);
var storeFactory = new MockRedisContentLocationStoreFactory(
localDatabase,
localMachineDatabase,
rootPath,
mockClock: TestClock,
_configuration);
var distributedContentStore = new DistributedContentStore <AbsolutePath>(
localMachineData,
(nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) =>
new FileSystemContentStore(
FileSystem,
TestClock,
rootPath,
configurationModel,
nagleQueue: nagleBlock,
distributedEvictionSettings: distributedEvictionSettings,
settings: contentStoreSettings,
trimBulkAsync: trimBulkAsync),
storeFactory,
fileCopier,
fileCopier,
pathTransformer,
copyRequester,
ContentAvailabilityGuarantee,
tempPath,
FileSystem,
RedisContentLocationStoreConstants.DefaultBatchSize,
retryIntervalForCopies: DistributedContentSessionTests.DefaultRetryIntervalsForTest,
replicaCreditInMinutes: replicaCreditInMinutes,
pinConfiguration: PinConfiguration,
clock: TestClock,
enableRepairHandling: enableRepairHandling,
contentStoreSettings: new ContentStoreSettings()
{
CheckFiles = true,
UseEmptyFileHashShortcut = emptyFileHashShortcutEnabled,
UseLegacyQuotaKeeperImplementation = false,
});
distributedContentStore.DisposeContentStoreFactory = false;
return(distributedContentStore);
}
