/// <nodoc />
public DistributedContentCopier(
AbsolutePath workingDirectory,
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IFileCopier <T> fileCopier,
IFileExistenceChecker <T> fileExistenceChecker,
ICopyRequester copyRequester,
IPathTransformer <T> pathTransformer,
IContentLocationStore contentLocationStore)
{
Contract.Requires(settings != null);
Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);
_settings = settings;
_tempFolderForCopies = new DisposableDirectory(fileSystem, workingDirectory / "Temp");
_remoteFileCopier = fileCopier;
_remoteFileExistenceChecker = fileExistenceChecker;
_copyRequester = copyRequester;
_contentLocationStore = contentLocationStore;
_pathTransformer = pathTransformer;
_fileSystem = fileSystem;
_workingDirectory = _tempFolderForCopies.Path;
_ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations);
_proactiveCopyIoGate = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations);
_retryIntervals = settings.RetryIntervalForCopies;
_maxRetryCount = settings.MaxRetryCount;
_timeoutForPoractiveCopies = settings.TimeoutForProactiveCopies;
}
/// <inheritdoc />
public Task <IContentLocationStore> CreateAsync(MachineLocation localMachineLocation)
{
IContentLocationStore contentLocationStore = null;
if (Configuration.HasReadOrWriteMode(ContentLocationMode.Redis))
{
var redisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForContent);
var machineLocationRedisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForMachineLocations);
contentLocationStore = new RedisContentLocationStore(
redisDatabaseAdapter,
machineLocationRedisDatabaseAdapter,
Clock,
_contentHashBumpTime,
localMachineLocation.Data,
Configuration);
}
if (Configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore))
{
Contract.Assert(RedisDatabaseFactoryForRedisGlobalStore != null);
var redisDatabaseForGlobalStore = CreateDatabase(RedisDatabaseFactoryForRedisGlobalStore);
var secondaryRedisDatabaseForGlobalStore = CreateDatabase(RedisDatabaseFactoryForRedisGlobalStoreSecondary, optional: true);
IGlobalLocationStore globalStore = new RedisGlobalStore(Clock, Configuration, localMachineLocation, redisDatabaseForGlobalStore, secondaryRedisDatabaseForGlobalStore);
var localLocationStore = new LocalLocationStore(Clock, globalStore, Configuration);
contentLocationStore = new TransitioningContentLocationStore(Configuration, (RedisContentLocationStore)contentLocationStore, localLocationStore, Clock);
}
return(Task.FromResult(contentLocationStore));
}
/// <inheritdoc />
public Task <IContentLocationStore> CreateAsync(MachineLocation localMachineLocation, ILocalContentStore localContentStore)
{
IContentLocationStore contentLocationStore = null;
if (Configuration.HasReadOrWriteMode(ContentLocationMode.Redis))
{
var redisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForContent, "RedisDatabaseFactoryForContent");
var machineLocationRedisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForMachineLocations, "RedisDatabaseFactoryForMachineLocations");
contentLocationStore = new RedisContentLocationStore(
redisDatabaseAdapter,
machineLocationRedisDatabaseAdapter,
Clock,
_contentHashBumpTime,
localMachineLocation.Data,
Configuration);
}
if (Configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore))
{
var localLocationStore = _lazyLocalLocationStore.Value;
contentLocationStore = new TransitioningContentLocationStore(Configuration, (RedisContentLocationStore)contentLocationStore, localLocationStore, localMachineLocation, localContentStore);
}
return(Task.FromResult(contentLocationStore));
}
/// <nodoc />
public DistributedContentCopier(
AbsolutePath workingDirectory,
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IFileCopier <T> fileCopier,
IFileExistenceChecker <T> fileExistenceChecker,
IPathTransformer <T> pathTransformer,
IContentLocationStore contentLocationStore)
{
Contract.Requires(settings != null);
Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);
_settings = settings;
_tempFolderForCopies = new DisposableDirectory(fileSystem, workingDirectory / "Temp");
_remoteFileCopier = fileCopier;
_remoteFileExistenceChecker = fileExistenceChecker;
_contentLocationStore = contentLocationStore;
_pathTransformer = pathTransformer;
_fileSystem = fileSystem;
_workingDirectory = _tempFolderForCopies.Path;
// TODO: Use hashers from IContentStoreInternal instead?
_hashers = HashInfoLookup.CreateAll();
_ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations);
_retryIntervals = settings.RetryIntervalForCopies;
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
await _distributedCopier.StartupAsync(context).ThrowIfFailure();
// NOTE: We create and start the content location store before the inner content store just in case the
// inner content store starts background eviction after startup. We need the content store to be initialized
// so that it can be queried and used to unregister content.
await _contentLocationStoreFactory.StartupAsync(context).ThrowIfFailure();
_contentLocationStore = await _contentLocationStoreFactory.CreateAsync(LocalMachineLocation, InnerContentStore as ILocalContentStore);
// Initializing inner store before initializing LocalLocationStore because
// LocalLocationStore may use inner store for reconciliation purposes
await InnerContentStore.StartupAsync(context).ThrowIfFailure();
await _contentLocationStore.StartupAsync(context).ThrowIfFailure();
if (_settings.EnableProactiveReplication &&
_contentLocationStore is TransitioningContentLocationStore tcs &&
InnerContentStore is ILocalContentStore localContentStore)
{
await ProactiveReplicationAsync(context.CreateNested(nameof(DistributedContentStore)), localContentStore, tcs).ThrowIfFailure();
}
return(BoolResult.Success);
}
/// <summary>
/// Retrieves the content locations for a given set of content hashes from local and global stores.
/// </summary>
public static IEnumerable <Task <GetBulkLocationsResult> > MultiLevelGetLocations(
this IContentLocationStore store,
Context context,
IReadOnlyList <ContentHash> contentHashes,
CancellationToken cts,
UrgencyHint urgencyHint,
bool subtractLocalResults)
{
var localResultsTask = store.GetBulkAsync(context, contentHashes, cts, urgencyHint, GetBulkOrigin.Local);
yield return(localResultsTask);
yield return(getBulkGlobal());
// Local function: Get global results optionally subtracting local results
async Task <GetBulkLocationsResult> getBulkGlobal()
{
var globalResults = await store.GetBulkAsync(context, contentHashes, cts, urgencyHint, GetBulkOrigin.Global);
if (subtractLocalResults)
{
globalResults = globalResults.Subtract(await localResultsTask);
}
return(globalResults);
}
}
public static Task <GetBulkLocationsResult> GetBulkAsync(
this IContentLocationStore store,
Context context,
ContentHash contentHash,
GetBulkOrigin origin)
{
return(store.GetBulkAsync(context, new ContentHash[] { contentHash }, CancellationToken.None, UrgencyHint.Nominal, origin));
}
/// <summary>
/// Retrieves the content locations for a given set of content hashes.
/// </summary>
public static Task <GetBulkLocationsResult> GetBulkAsync(
this IContentLocationStore store,
Context context,
IReadOnlyList <ContentHash> contentHashes,
CancellationToken cts,
UrgencyHint urgencyHint)
{
return(store.GetBulkAsync(context, contentHashes, cts, urgencyHint, GetBulkOrigin.Global));
}
public TestDistributedContentCopier(
AbsolutePath workingDirectory,
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IFileCopier <AbsolutePath> fileCopier,
IFileExistenceChecker <AbsolutePath> fileExistenceChecker,
IContentCommunicationManager copyRequester,
IPathTransformer <AbsolutePath> pathTransformer,
IContentLocationStore contentLocationStore)
: base(workingDirectory, settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransformer, TestSystemClock.Instance, contentLocationStore)
{
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
// NOTE: We create and start the content location store before the inner content store just in case the
// inner content store starts background eviction after startup. We need the content store to be initialized
// so that it can be queried and used to unregister content.
await _contentLocationStoreFactory.StartupAsync(context).ThrowIfFailure();
_contentLocationStore = await _contentLocationStoreFactory.CreateAsync();
_distributedCopier = _distributedCopierFactory(_contentLocationStore);
await _distributedCopier.StartupAsync(context).ThrowIfFailure();
if (_contentLocationStore is TransitioningContentLocationStore tcs)
{
tcs.LocalLocationStore.PreStartupInitialize(context, InnerContentStore as ILocalContentStore, _distributedCopier);
}
// Initializing inner store before initializing LocalLocationStore because
// LocalLocationStore may use inner store for reconciliation purposes
await InnerContentStore.StartupAsync(context).ThrowIfFailure();
await _contentLocationStore.StartupAsync(context).ThrowIfFailure();
Func <ContentHash[], Task> evictionHandler;
var localContext = new Context(context);
if (_enableDistributedEviction)
{
evictionHandler = hashes => EvictContentAsync(localContext, hashes);
}
else
{
evictionHandler = hashes => DistributedGarbageCollectionAsync(localContext, hashes);
}
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup. So we start the queue here.
_evictionNagleQueue.Start(evictionHandler);
var touchContext = new Context(context);
_touchNagleQueue = NagleQueue <ContentHashWithSize> .Create(
hashes => TouchBulkAsync(touchContext, hashes),
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
batchSize : _locationStoreBatchSize);
return(BoolResult.Success);
}
/// <nodoc />
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
DistributedContentCopier contentCopier,
IDistributedContentCopierHost copierHost,
MachineLocation localMachineLocation,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentCopier,
copierHost,
localMachineLocation,
settings)
{
}
/// <summary>
/// Retrieves the content locations for a given set of content hashes where Global origin returns merged global and local content locations.
/// </summary>
public static async Task <GetBulkLocationsResult> GetBulkStackedAsync(
this IContentLocationStore store,
Context context,
IReadOnlyList <ContentHash> contentHashes,
CancellationToken cts,
UrgencyHint urgencyHint,
GetBulkOrigin origin)
{
var localResults = await store.GetBulkAsync(context, contentHashes, cts, urgencyHint, GetBulkOrigin.Local);
if (origin == GetBulkOrigin.Local)
{
return(localResults);
}
var globalResults = await store.GetBulkAsync(context, contentHashes, cts, urgencyHint, GetBulkOrigin.Global);
return(localResults.Merge(globalResults));
}
/// <nodoc />
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
DistributedContentCopier contentCopier,
DistributedContentStore distributedStore,
MachineLocation localMachineLocation,
ColdStorage coldStorage,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentCopier,
distributedStore,
localMachineLocation,
coldStorage,
settings)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class.
/// </summary>
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
DistributedContentCopier <T> contentCopier,
MachineLocation localMachineLocation,
PinCache pinCache = null,
ContentTrackerUpdater contentTrackerUpdater = null,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentCopier,
localMachineLocation,
pinCache: pinCache,
contentTrackerUpdater: contentTrackerUpdater,
settings)
{
_putFileGate = new SemaphoreSlim(settings.MaximumConcurrentPutFileOperations);
}
/// <summary>
/// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class.
/// </summary>
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
DistributedContentCopier <T> contentCopier,
IDistributedContentCopierHost copierHost,
MachineLocation localMachineLocation,
PinCache pinCache = null,
ContentTrackerUpdater contentTrackerUpdater = null,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentCopier,
copierHost,
localMachineLocation,
pinCache: pinCache,
contentTrackerUpdater: contentTrackerUpdater,
settings)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class.
/// </summary>
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
ContentAvailabilityGuarantee contentAvailabilityGuarantee,
DistributedContentCopier <T> contentCopier,
byte[] localMachineLocation,
PinCache pinCache = null,
ContentTrackerUpdater contentTrackerUpdater = null,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentAvailabilityGuarantee,
contentCopier,
localMachineLocation,
pinCache: pinCache,
contentTrackerUpdater: contentTrackerUpdater,
settings)
{
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
// NOTE: We create and start the content location store before the inner content store just in case the
// inner content store starts background eviction after startup. We need the content store to be initialized
// so that it can be queried and used to unregister content.
await _contentLocationStoreFactory.StartupAsync(context).ThrowIfFailure();
_contentLocationStore = await _contentLocationStoreFactory.CreateAsync(LocalMachineLocation);
_distributedCopier = _distributedCopierFactory(_contentLocationStore);
await _distributedCopier.StartupAsync(context).ThrowIfFailure();
if (_contentLocationStore is TransitioningContentLocationStore tcs && tcs.IsLocalLocationStoreEnabled)
{
// Define proactive copy logic.
async Task <ResultBase> proactiveCopyTaskFactory(OperationContext operationContext, ContentHash hash)
{
var sessionResult = await _proactiveCopySession.Value;
if (sessionResult)
{
return(await sessionResult.Value.ProactiveCopyIfNeededAsync(operationContext, hash, tryBuildRing : false));
}
return(new BoolResult("Failed to retrieve session for proactive copies."));
}
tcs.LocalLocationStore.PreStartupInitialize(context, InnerContentStore as ILocalContentStore, _distributedCopier, proactiveCopyTaskFactory);
}
// Initializing inner store before initializing LocalLocationStore because
// LocalLocationStore may use inner store for reconciliation purposes
await InnerContentStore.StartupAsync(context).ThrowIfFailure();
await _contentLocationStore.StartupAsync(context).ThrowIfFailure();
Func <ContentHash[], Task> evictionHandler;
var localContext = context.CreateNested();
if (_enableDistributedEviction)
{
evictionHandler = hashes => EvictContentAsync(localContext, hashes);
}
else
{
evictionHandler = hashes => DistributedGarbageCollectionAsync(localContext, hashes);
}
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup. So we start the queue here.
_evictionNagleQueue.Start(evictionHandler);
var touchContext = context.CreateNested();
_touchNagleQueue = NagleQueue <ContentHashWithSize> .Create(
hashes => TouchBulkAsync(touchContext, hashes),
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
batchSize : _locationStoreBatchSize);
return(BoolResult.Success);
}
