public ColdStorage(IAbsFileSystem fileSystem, ColdStorageSettings coldStorageSettings, DistributedContentCopier distributedContentCopier) { _fileSystem = fileSystem; _copier = distributedContentCopier; _rootPath = coldStorageSettings.GetAbsoulutePath(); ConfigurationModel configurationModel = new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota(coldStorageSettings.CacheSizeQuotaString !))); ContentStoreSettings contentStoreSettings = FromColdStorageSettings(coldStorageSettings); _store = new FileSystemContentStore(fileSystem, SystemClock.Instance, _rootPath, configurationModel, null, contentStoreSettings, null); HashType hashType; if (!Enum.TryParse <HashType>(coldStorageSettings.ConsistentHashingHashType, true, out hashType)) { hashType = HashType.SHA256; } _contentHasher = HashInfoLookup.GetContentHasher(hashType); _copiesQuantity = coldStorageSettings.ConsistentHashingCopiesQuantity; _maxParallelPlaces = coldStorageSettings.MaxParallelPlaces; // Starts empty and is created during the first update _ring = new RingNode[0]; }
/// <inheritdoc /> protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { // NOTE: We create and start the content location store before the inner content store just in case the // inner content store starts background eviction after startup. We need the content store to be initialized // so that it can be queried and used to unregister content. await _contentLocationStoreFactory.StartupAsync(context).ThrowIfFailure(); _contentLocationStore = await _contentLocationStoreFactory.CreateAsync(); _distributedCopier = _distributedCopierFactory(_contentLocationStore); await _distributedCopier.StartupAsync(context).ThrowIfFailure(); if (_contentLocationStore is TransitioningContentLocationStore tcs) { tcs.LocalLocationStore.PreStartupInitialize(context, InnerContentStore as ILocalContentStore, _distributedCopier); } // Initializing inner store before initializing LocalLocationStore because // LocalLocationStore may use inner store for reconciliation purposes await InnerContentStore.StartupAsync(context).ThrowIfFailure(); await _contentLocationStore.StartupAsync(context).ThrowIfFailure(); Func <ContentHash[], Task> evictionHandler; var localContext = new Context(context); if (_enableDistributedEviction) { evictionHandler = hashes => EvictContentAsync(localContext, hashes); } else { evictionHandler = hashes => DistributedGarbageCollectionAsync(localContext, hashes); } // Queue is created in unstarted state because the eviction function // requires the context passed at startup. So we start the queue here. _evictionNagleQueue.Start(evictionHandler); var touchContext = new Context(context); _touchNagleQueue = NagleQueue <ContentHashWithSize> .Create( hashes => TouchBulkAsync(touchContext, hashes), Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism, Redis.RedisContentLocationStoreConstants.BatchInterval, batchSize : _locationStoreBatchSize); return(BoolResult.Success); }
/// <nodoc /> public DistributedContentStore( MachineLocation localMachineLocation, AbsolutePath localCacheRoot, Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, DistributedContentStoreSettings settings, DistributedContentCopier <T> distributedCopier, IClock clock = null, ContentStoreSettings contentStoreSettings = null) { Contract.Requires(settings != null); LocalMachineLocation = localMachineLocation; _contentLocationStoreFactory = contentLocationStoreFactory; _clock = clock; _distributedCopier = distributedCopier; _copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp"); contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings; _settings = settings; // Queue is created in unstarted state because the eviction function // requires the context passed at startup. _evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted( Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism, Redis.RedisContentLocationStoreConstants.BatchInterval, _settings.LocationStoreBatchSize); _enableDistributedEviction = _settings.ReplicaCreditInMinutes != null; var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(_settings.ReplicaCreditInMinutes, _settings.LocationStoreBatchSize) : null; var enableTouch = _settings.ContentHashBumpTime.HasValue; if (enableTouch) { _contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, _settings.ContentHashBumpTime.Value, clock: _clock); } TrimBulkAsync trimBulkAsync = null; InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync); if (settings.PinConfiguration?.IsPinCachingEnabled == true) { _pinCache = new PinCache(clock: _clock); } }
public DistributedContentStore( MachineLocation localMachineLocation, AbsolutePath localCacheRoot, Func <IDistributedLocationStore, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, DistributedContentStoreSettings settings, DistributedContentCopier distributedCopier, IClock?clock = null) { Contract.Requires(settings != null); LocalMachineLocation = localMachineLocation; _contentLocationStoreFactory = contentLocationStoreFactory; _clock = clock ?? SystemClock.Instance; _distributedCopier = distributedCopier; _copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp"); _settings = settings; InnerContentStore = innerContentStoreFunc(this); }
/// <nodoc /> public DistributedContentStore( MachineLocation localMachineLocation, AbsolutePath localCacheRoot, Func <ContentStoreSettings, IDistributedLocationStore, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, DistributedContentStoreSettings settings, DistributedContentCopier <T> distributedCopier, IClock clock = null, ContentStoreSettings contentStoreSettings = null) { Contract.Requires(settings != null); LocalMachineLocation = localMachineLocation; _contentLocationStoreFactory = contentLocationStoreFactory; _clock = clock; _distributedCopier = distributedCopier; _copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp"); contentStoreSettings ??= ContentStoreSettings.DefaultSettings; _settings = settings; InnerContentStore = innerContentStoreFunc(contentStoreSettings, this); }
/// <inheritdoc /> protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { // NOTE: We create and start the content location store before the inner content store just in case the // inner content store starts background eviction after startup. We need the content store to be initialized // so that it can be queried and used to unregister content. await _contentLocationStoreFactory.StartupAsync(context).ThrowIfFailure(); _contentLocationStore = await _contentLocationStoreFactory.CreateAsync(LocalMachineLocation); _distributedCopier = _distributedCopierFactory(_contentLocationStore); await _distributedCopier.StartupAsync(context).ThrowIfFailure(); if (_contentLocationStore is TransitioningContentLocationStore tcs && tcs.IsLocalLocationStoreEnabled) { // Define proactive copy logic. async Task <ResultBase> proactiveCopyTaskFactory(OperationContext operationContext, ContentHash hash) { var sessionResult = await _proactiveCopySession.Value; if (sessionResult) { return(await sessionResult.Value.ProactiveCopyIfNeededAsync(operationContext, hash, tryBuildRing : false)); } return(new BoolResult("Failed to retrieve session for proactive copies.")); } tcs.LocalLocationStore.PreStartupInitialize(context, InnerContentStore as ILocalContentStore, _distributedCopier, proactiveCopyTaskFactory); } // Initializing inner store before initializing LocalLocationStore because // LocalLocationStore may use inner store for reconciliation purposes await InnerContentStore.StartupAsync(context).ThrowIfFailure(); await _contentLocationStore.StartupAsync(context).ThrowIfFailure(); Func <ContentHash[], Task> evictionHandler; var localContext = context.CreateNested(); if (_enableDistributedEviction) { evictionHandler = hashes => EvictContentAsync(localContext, hashes); } else { evictionHandler = hashes => DistributedGarbageCollectionAsync(localContext, hashes); } // Queue is created in unstarted state because the eviction function // requires the context passed at startup. So we start the queue here. _evictionNagleQueue.Start(evictionHandler); var touchContext = context.CreateNested(); _touchNagleQueue = NagleQueue <ContentHashWithSize> .Create( hashes => TouchBulkAsync(touchContext, hashes), Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism, Redis.RedisContentLocationStoreConstants.BatchInterval, batchSize : _locationStoreBatchSize); return(BoolResult.Success); }