/// <nodoc /> public DistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IFileCopier <T> fileCopier, IFileExistenceChecker <T> fileExistenceChecker, ICopyRequester copyRequester, IPathTransformer <T> pathTransformer, IContentLocationStore contentLocationStore) { Contract.Requires(settings != null); Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _tempFolderForCopies = new DisposableDirectory(fileSystem, workingDirectory / "Temp"); _remoteFileCopier = fileCopier; _remoteFileExistenceChecker = fileExistenceChecker; _copyRequester = copyRequester; _contentLocationStore = contentLocationStore; _pathTransformer = pathTransformer; _fileSystem = fileSystem; _workingDirectory = _tempFolderForCopies.Path; _ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations); _proactiveCopyIoGate = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations); _retryIntervals = settings.RetryIntervalForCopies; _maxRetryCount = settings.MaxRetryCount; _timeoutForPoractiveCopies = settings.TimeoutForProactiveCopies; }
/// <inheritdoc /> public Task <IContentLocationStore> CreateAsync(MachineLocation localMachineLocation) { IContentLocationStore contentLocationStore = null; if (Configuration.HasReadOrWriteMode(ContentLocationMode.Redis)) { var redisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForContent); var machineLocationRedisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForMachineLocations); contentLocationStore = new RedisContentLocationStore( redisDatabaseAdapter, machineLocationRedisDatabaseAdapter, Clock, _contentHashBumpTime, localMachineLocation.Data, Configuration); } if (Configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore)) { Contract.Assert(RedisDatabaseFactoryForRedisGlobalStore != null); var redisDatabaseForGlobalStore = CreateDatabase(RedisDatabaseFactoryForRedisGlobalStore); var secondaryRedisDatabaseForGlobalStore = CreateDatabase(RedisDatabaseFactoryForRedisGlobalStoreSecondary, optional: true); IGlobalLocationStore globalStore = new RedisGlobalStore(Clock, Configuration, localMachineLocation, redisDatabaseForGlobalStore, secondaryRedisDatabaseForGlobalStore); var localLocationStore = new LocalLocationStore(Clock, globalStore, Configuration); contentLocationStore = new TransitioningContentLocationStore(Configuration, (RedisContentLocationStore)contentLocationStore, localLocationStore, Clock); } return(Task.FromResult(contentLocationStore)); }
/// <inheritdoc /> public Task <IContentLocationStore> CreateAsync(MachineLocation localMachineLocation, ILocalContentStore localContentStore) { IContentLocationStore contentLocationStore = null; if (Configuration.HasReadOrWriteMode(ContentLocationMode.Redis)) { var redisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForContent, "RedisDatabaseFactoryForContent"); var machineLocationRedisDatabaseAdapter = CreateDatabase(RedisDatabaseFactoryForMachineLocations, "RedisDatabaseFactoryForMachineLocations"); contentLocationStore = new RedisContentLocationStore( redisDatabaseAdapter, machineLocationRedisDatabaseAdapter, Clock, _contentHashBumpTime, localMachineLocation.Data, Configuration); } if (Configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore)) { var localLocationStore = _lazyLocalLocationStore.Value; contentLocationStore = new TransitioningContentLocationStore(Configuration, (RedisContentLocationStore)contentLocationStore, localLocationStore, localMachineLocation, localContentStore); } return(Task.FromResult(contentLocationStore)); }
/// <nodoc /> public DistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IFileCopier <T> fileCopier, IFileExistenceChecker <T> fileExistenceChecker, IPathTransformer <T> pathTransformer, IContentLocationStore contentLocationStore) { Contract.Requires(settings != null); Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _tempFolderForCopies = new DisposableDirectory(fileSystem, workingDirectory / "Temp"); _remoteFileCopier = fileCopier; _remoteFileExistenceChecker = fileExistenceChecker; _contentLocationStore = contentLocationStore; _pathTransformer = pathTransformer; _fileSystem = fileSystem; _workingDirectory = _tempFolderForCopies.Path; // TODO: Use hashers from IContentStoreInternal instead? _hashers = HashInfoLookup.CreateAll(); _ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations); _retryIntervals = settings.RetryIntervalForCopies; }
/// <inheritdoc /> protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { await _distributedCopier.StartupAsync(context).ThrowIfFailure(); // NOTE: We create and start the content location store before the inner content store just in case the // inner content store starts background eviction after startup. We need the content store to be initialized // so that it can be queried and used to unregister content. await _contentLocationStoreFactory.StartupAsync(context).ThrowIfFailure(); _contentLocationStore = await _contentLocationStoreFactory.CreateAsync(LocalMachineLocation, InnerContentStore as ILocalContentStore); // Initializing inner store before initializing LocalLocationStore because // LocalLocationStore may use inner store for reconciliation purposes await InnerContentStore.StartupAsync(context).ThrowIfFailure(); await _contentLocationStore.StartupAsync(context).ThrowIfFailure(); if (_settings.EnableProactiveReplication && _contentLocationStore is TransitioningContentLocationStore tcs && InnerContentStore is ILocalContentStore localContentStore) { await ProactiveReplicationAsync(context.CreateNested(nameof(DistributedContentStore)), localContentStore, tcs).ThrowIfFailure(); } return(BoolResult.Success); }
/// <summary> /// Retrieves the content locations for a given set of content hashes from local and global stores. /// </summary> public static IEnumerable <Task <GetBulkLocationsResult> > MultiLevelGetLocations( this IContentLocationStore store, Context context, IReadOnlyList <ContentHash> contentHashes, CancellationToken cts, UrgencyHint urgencyHint, bool subtractLocalResults) { var localResultsTask = store.GetBulkAsync(context, contentHashes, cts, urgencyHint, GetBulkOrigin.Local); yield return(localResultsTask); yield return(getBulkGlobal()); // Local function: Get global results optionally subtracting local results async Task <GetBulkLocationsResult> getBulkGlobal() { var globalResults = await store.GetBulkAsync(context, contentHashes, cts, urgencyHint, GetBulkOrigin.Global); if (subtractLocalResults) { globalResults = globalResults.Subtract(await localResultsTask); } return(globalResults); } }
public static Task <GetBulkLocationsResult> GetBulkAsync( this IContentLocationStore store, Context context, ContentHash contentHash, GetBulkOrigin origin) { return(store.GetBulkAsync(context, new ContentHash[] { contentHash }, CancellationToken.None, UrgencyHint.Nominal, origin)); }
/// <summary> /// Retrieves the content locations for a given set of content hashes. /// </summary> public static Task <GetBulkLocationsResult> GetBulkAsync( this IContentLocationStore store, Context context, IReadOnlyList <ContentHash> contentHashes, CancellationToken cts, UrgencyHint urgencyHint) { return(store.GetBulkAsync(context, contentHashes, cts, urgencyHint, GetBulkOrigin.Global)); }
public TestDistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IFileCopier <AbsolutePath> fileCopier, IFileExistenceChecker <AbsolutePath> fileExistenceChecker, IContentCommunicationManager copyRequester, IPathTransformer <AbsolutePath> pathTransformer, IContentLocationStore contentLocationStore) : base(workingDirectory, settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransformer, TestSystemClock.Instance, contentLocationStore) { }
/// <inheritdoc /> protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { // NOTE: We create and start the content location store before the inner content store just in case the // inner content store starts background eviction after startup. We need the content store to be initialized // so that it can be queried and used to unregister content. await _contentLocationStoreFactory.StartupAsync(context).ThrowIfFailure(); _contentLocationStore = await _contentLocationStoreFactory.CreateAsync(); _distributedCopier = _distributedCopierFactory(_contentLocationStore); await _distributedCopier.StartupAsync(context).ThrowIfFailure(); if (_contentLocationStore is TransitioningContentLocationStore tcs) { tcs.LocalLocationStore.PreStartupInitialize(context, InnerContentStore as ILocalContentStore, _distributedCopier); } // Initializing inner store before initializing LocalLocationStore because // LocalLocationStore may use inner store for reconciliation purposes await InnerContentStore.StartupAsync(context).ThrowIfFailure(); await _contentLocationStore.StartupAsync(context).ThrowIfFailure(); Func <ContentHash[], Task> evictionHandler; var localContext = new Context(context); if (_enableDistributedEviction) { evictionHandler = hashes => EvictContentAsync(localContext, hashes); } else { evictionHandler = hashes => DistributedGarbageCollectionAsync(localContext, hashes); } // Queue is created in unstarted state because the eviction function // requires the context passed at startup. So we start the queue here. _evictionNagleQueue.Start(evictionHandler); var touchContext = new Context(context); _touchNagleQueue = NagleQueue <ContentHashWithSize> .Create( hashes => TouchBulkAsync(touchContext, hashes), Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism, Redis.RedisContentLocationStoreConstants.BatchInterval, batchSize : _locationStoreBatchSize); return(BoolResult.Success); }
/// <nodoc /> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier contentCopier, IDistributedContentCopierHost copierHost, MachineLocation localMachineLocation, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentCopier, copierHost, localMachineLocation, settings) { }
/// <summary> /// Retrieves the content locations for a given set of content hashes where Global origin returns merged global and local content locations. /// </summary> public static async Task <GetBulkLocationsResult> GetBulkStackedAsync( this IContentLocationStore store, Context context, IReadOnlyList <ContentHash> contentHashes, CancellationToken cts, UrgencyHint urgencyHint, GetBulkOrigin origin) { var localResults = await store.GetBulkAsync(context, contentHashes, cts, urgencyHint, GetBulkOrigin.Local); if (origin == GetBulkOrigin.Local) { return(localResults); } var globalResults = await store.GetBulkAsync(context, contentHashes, cts, urgencyHint, GetBulkOrigin.Global); return(localResults.Merge(globalResults)); }
/// <nodoc /> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier contentCopier, DistributedContentStore distributedStore, MachineLocation localMachineLocation, ColdStorage coldStorage, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentCopier, distributedStore, localMachineLocation, coldStorage, settings) { }
/// <summary> /// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class. /// </summary> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier <T> contentCopier, MachineLocation localMachineLocation, PinCache pinCache = null, ContentTrackerUpdater contentTrackerUpdater = null, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentCopier, localMachineLocation, pinCache: pinCache, contentTrackerUpdater: contentTrackerUpdater, settings) { _putFileGate = new SemaphoreSlim(settings.MaximumConcurrentPutFileOperations); }
/// <summary> /// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class. /// </summary> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier <T> contentCopier, IDistributedContentCopierHost copierHost, MachineLocation localMachineLocation, PinCache pinCache = null, ContentTrackerUpdater contentTrackerUpdater = null, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentCopier, copierHost, localMachineLocation, pinCache: pinCache, contentTrackerUpdater: contentTrackerUpdater, settings) { }
/// <summary> /// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class. /// </summary> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, ContentAvailabilityGuarantee contentAvailabilityGuarantee, DistributedContentCopier <T> contentCopier, byte[] localMachineLocation, PinCache pinCache = null, ContentTrackerUpdater contentTrackerUpdater = null, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentAvailabilityGuarantee, contentCopier, localMachineLocation, pinCache: pinCache, contentTrackerUpdater: contentTrackerUpdater, settings) { }
/// <inheritdoc /> protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { // NOTE: We create and start the content location store before the inner content store just in case the // inner content store starts background eviction after startup. We need the content store to be initialized // so that it can be queried and used to unregister content. await _contentLocationStoreFactory.StartupAsync(context).ThrowIfFailure(); _contentLocationStore = await _contentLocationStoreFactory.CreateAsync(LocalMachineLocation); _distributedCopier = _distributedCopierFactory(_contentLocationStore); await _distributedCopier.StartupAsync(context).ThrowIfFailure(); if (_contentLocationStore is TransitioningContentLocationStore tcs && tcs.IsLocalLocationStoreEnabled) { // Define proactive copy logic. async Task <ResultBase> proactiveCopyTaskFactory(OperationContext operationContext, ContentHash hash) { var sessionResult = await _proactiveCopySession.Value; if (sessionResult) { return(await sessionResult.Value.ProactiveCopyIfNeededAsync(operationContext, hash, tryBuildRing : false)); } return(new BoolResult("Failed to retrieve session for proactive copies.")); } tcs.LocalLocationStore.PreStartupInitialize(context, InnerContentStore as ILocalContentStore, _distributedCopier, proactiveCopyTaskFactory); } // Initializing inner store before initializing LocalLocationStore because // LocalLocationStore may use inner store for reconciliation purposes await InnerContentStore.StartupAsync(context).ThrowIfFailure(); await _contentLocationStore.StartupAsync(context).ThrowIfFailure(); Func <ContentHash[], Task> evictionHandler; var localContext = context.CreateNested(); if (_enableDistributedEviction) { evictionHandler = hashes => EvictContentAsync(localContext, hashes); } else { evictionHandler = hashes => DistributedGarbageCollectionAsync(localContext, hashes); } // Queue is created in unstarted state because the eviction function // requires the context passed at startup. So we start the queue here. _evictionNagleQueue.Start(evictionHandler); var touchContext = context.CreateNested(); _touchNagleQueue = NagleQueue <ContentHashWithSize> .Create( hashes => TouchBulkAsync(touchContext, hashes), Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism, Redis.RedisContentLocationStoreConstants.BatchInterval, batchSize : _locationStoreBatchSize); return(BoolResult.Success); }