public DistributedContentCopier( DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IFileCopier <T> fileCopier, IFileExistenceChecker <T> fileExistenceChecker, IContentCommunicationManager copyRequester, IPathTransformer <T> pathTransformer, IClock clock) { Contract.Requires(settings != null); Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _remoteFileCopier = fileCopier; _remoteFileExistenceChecker = fileExistenceChecker; _copyRequester = copyRequester; _pathTransformer = pathTransformer; FileSystem = fileSystem; _clock = clock; _ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations); _proactiveCopyIoGate = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations); _retryIntervals = settings.RetryIntervalForCopies; _maxRetryCount = settings.MaxRetryCount; _timeoutForProactiveCopies = settings.TimeoutForProactiveCopies; }
public DistributedContentCopier( DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IRemoteFileCopier fileCopier, IContentCommunicationManager copyRequester, IClock clock, ILogger logger) { Contract.Requires(settings != null); Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _remoteFileCopier = fileCopier; _copyRequester = copyRequester; FileSystem = fileSystem; _clock = clock; var context = new Context(logger); _copyScheduler = settings.CopyScheduler.Create(context); if (_copyRequester is StartupShutdownSlimBase slimBase) { slimBase.StartupAsync(context).Result.ThrowIfFailure(); } _retryIntervals = settings.RetryIntervalForCopies; _maxRetryCount = settings.MaxRetryCount; }
/// <nodoc /> public DistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IFileCopier <T> fileCopier, IFileExistenceChecker <T> fileExistenceChecker, ICopyRequester copyRequester, IPathTransformer <T> pathTransformer, IContentLocationStore contentLocationStore) { Contract.Requires(settings != null); Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _tempFolderForCopies = new DisposableDirectory(fileSystem, workingDirectory / "Temp"); _remoteFileCopier = fileCopier; _remoteFileExistenceChecker = fileExistenceChecker; _copyRequester = copyRequester; _contentLocationStore = contentLocationStore; _pathTransformer = pathTransformer; _fileSystem = fileSystem; _workingDirectory = _tempFolderForCopies.Path; _ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations); _proactiveCopyIoGate = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations); _retryIntervals = settings.RetryIntervalForCopies; _maxRetryCount = settings.MaxRetryCount; _timeoutForPoractiveCopies = settings.TimeoutForProactiveCopies; }
/// <nodoc /> public DistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IFileCopier <T> fileCopier, IFileExistenceChecker <T> fileExistenceChecker, IPathTransformer <T> pathTransformer, IContentLocationStore contentLocationStore) { Contract.Requires(settings != null); Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _tempFolderForCopies = new DisposableDirectory(fileSystem, workingDirectory / "Temp"); _remoteFileCopier = fileCopier; _remoteFileExistenceChecker = fileExistenceChecker; _contentLocationStore = contentLocationStore; _pathTransformer = pathTransformer; _fileSystem = fileSystem; _workingDirectory = _tempFolderForCopies.Path; // TODO: Use hashers from IContentStoreInternal instead? _hashers = HashInfoLookup.CreateAll(); _ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations); _retryIntervals = settings.RetryIntervalForCopies; }
/// <nodoc /> public DistributedContentStore( MachineLocation localMachineLocation, AbsolutePath localCacheRoot, Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, DistributedContentStoreSettings settings, DistributedContentCopier <T> distributedCopier, IClock clock = null, ContentStoreSettings contentStoreSettings = null) { Contract.Requires(settings != null); LocalMachineLocation = localMachineLocation; _contentLocationStoreFactory = contentLocationStoreFactory; _clock = clock; _distributedCopier = distributedCopier; _copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp"); contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings; _settings = settings; // Queue is created in unstarted state because the eviction function // requires the context passed at startup. _evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted( Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism, Redis.RedisContentLocationStoreConstants.BatchInterval, _settings.LocationStoreBatchSize); _enableDistributedEviction = _settings.ReplicaCreditInMinutes != null; var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(_settings.ReplicaCreditInMinutes, _settings.LocationStoreBatchSize) : null; var enableTouch = _settings.ContentHashBumpTime.HasValue; if (enableTouch) { _contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, _settings.ContentHashBumpTime.Value, clock: _clock); } TrimBulkAsync trimBulkAsync = null; InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync); if (settings.PinConfiguration?.IsPinCachingEnabled == true) { _pinCache = new PinCache(clock: _clock); } }
public DistributedContentStore( MachineLocation localMachineLocation, AbsolutePath localCacheRoot, Func <IDistributedLocationStore, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, DistributedContentStoreSettings settings, DistributedContentCopier distributedCopier, IClock?clock = null) { Contract.Requires(settings != null); LocalMachineLocation = localMachineLocation; _contentLocationStoreFactory = contentLocationStoreFactory; _clock = clock ?? SystemClock.Instance; _distributedCopier = distributedCopier; _copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp"); _settings = settings; InnerContentStore = innerContentStoreFunc(this); }
public DistributedContentCopier( DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IRemoteFileCopier fileCopier, IContentCommunicationManager copyRequester, IClock clock, ILogger logger) { Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _remoteFileCopier = fileCopier; CommunicationManager = copyRequester; FileSystem = fileSystem; _clock = clock; Context = new Context(logger); _copyScheduler = settings.CopyScheduler.Create(Context); _retryIntervals = settings.RetryIntervalForCopies; _maxRetryCount = settings.MaxRetryCount; }
/// <nodoc /> public DistributedContentStore( MachineLocation localMachineLocation, AbsolutePath localCacheRoot, Func <ContentStoreSettings, IDistributedLocationStore, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, DistributedContentStoreSettings settings, DistributedContentCopier <T> distributedCopier, IClock clock = null, ContentStoreSettings contentStoreSettings = null) { Contract.Requires(settings != null); LocalMachineLocation = localMachineLocation; _contentLocationStoreFactory = contentLocationStoreFactory; _clock = clock; _distributedCopier = distributedCopier; _copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp"); contentStoreSettings ??= ContentStoreSettings.DefaultSettings; _settings = settings; InnerContentStore = innerContentStoreFunc(contentStoreSettings, this); }
public DistributedContentCopier( DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IRemoteFileCopier fileCopier, IContentCommunicationManager copyRequester, IClock clock, ILogger logger) { Contract.Requires(settings != null); Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _remoteFileCopier = fileCopier; _copyRequester = copyRequester; FileSystem = fileSystem; _clock = clock; _ioGate = new OrderedSemaphore(_settings.MaxConcurrentCopyOperations, _settings.OrderForCopies, new Context(logger)); _proactiveCopyIoGate = new OrderedSemaphore(_settings.MaxConcurrentProactiveCopyOperations, _settings.OrderForProactiveCopies, new Context(logger)); _retryIntervals = settings.RetryIntervalForCopies; _maxRetryCount = settings.MaxRetryCount; _ioGateTimeoutForProactiveCopies = settings.ProactiveCopyIOGateTimeout; }
/// <nodoc /> public DistributedContentStore( byte[] localMachineLocation, Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, IFileExistenceChecker <T> fileExistenceChecker, IFileCopier <T> fileCopier, IPathTransformer <T> pathTransform, ICopyRequester copyRequester, ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee, AbsolutePath tempFolderForCopies, IAbsFileSystem fileSystem, int locationStoreBatchSize, DistributedContentStoreSettings settings, int?replicaCreditInMinutes = null, IClock clock = null, bool enableRepairHandling = false, TimeSpan?contentHashBumpTime = null, ContentStoreSettings contentStoreSettings = null) { Contract.Requires(settings != null); LocalMachineLocation = new MachineLocation(localMachineLocation); _enableRepairHandling = enableRepairHandling; _contentLocationStoreFactory = contentLocationStoreFactory; _contentAvailabilityGuarantee = contentAvailabilityGuarantee; _locationStoreBatchSize = locationStoreBatchSize; contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings; _settings = settings; // Queue is created in unstarted state because the eviction function // requires the context passed at startup. _evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted( Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism, Redis.RedisContentLocationStoreConstants.BatchInterval, _locationStoreBatchSize); _distributedCopierFactory = (contentLocationStore) => { return(new DistributedContentCopier <T>( tempFolderForCopies, _settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransform, contentLocationStore)); }; _enableDistributedEviction = replicaCreditInMinutes != null; var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(replicaCreditInMinutes, locationStoreBatchSize) : null; var enableTouch = contentHashBumpTime.HasValue; if (enableTouch) { _contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, contentHashBumpTime.Value, clock: clock); } TrimBulkAsync trimBulkAsync = null; InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync); if (settings.PinConfiguration?.UsePinCache == true) { _pinCache = new PinCache(clock: clock); } }