Пример #1
0
        public DistributedContentCopier(
            DistributedContentStoreSettings settings,
            IAbsFileSystem fileSystem,
            IFileCopier <T> fileCopier,
            IFileExistenceChecker <T> fileExistenceChecker,
            IContentCommunicationManager copyRequester,
            IPathTransformer <T> pathTransformer,
            IClock clock)
        {
            Contract.Requires(settings != null);
            Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);

            _settings                   = settings;
            _remoteFileCopier           = fileCopier;
            _remoteFileExistenceChecker = fileExistenceChecker;
            _copyRequester              = copyRequester;
            _pathTransformer            = pathTransformer;
            FileSystem                  = fileSystem;
            _clock = clock;

            _ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations);
            _proactiveCopyIoGate       = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations);
            _retryIntervals            = settings.RetryIntervalForCopies;
            _maxRetryCount             = settings.MaxRetryCount;
            _timeoutForProactiveCopies = settings.TimeoutForProactiveCopies;
        }
        public DistributedContentCopier(
            DistributedContentStoreSettings settings,
            IAbsFileSystem fileSystem,
            IRemoteFileCopier fileCopier,
            IContentCommunicationManager copyRequester,
            IClock clock,
            ILogger logger)
        {
            Contract.Requires(settings != null);
            Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);

            _settings         = settings;
            _remoteFileCopier = fileCopier;
            _copyRequester    = copyRequester;
            FileSystem        = fileSystem;
            _clock            = clock;

            var context = new Context(logger);

            _copyScheduler = settings.CopyScheduler.Create(context);
            if (_copyRequester is StartupShutdownSlimBase slimBase)
            {
                slimBase.StartupAsync(context).Result.ThrowIfFailure();
            }

            _retryIntervals = settings.RetryIntervalForCopies;
            _maxRetryCount  = settings.MaxRetryCount;
        }
Пример #3
0
        /// <nodoc />
        public DistributedContentCopier(
            AbsolutePath workingDirectory,
            DistributedContentStoreSettings settings,
            IAbsFileSystem fileSystem,
            IFileCopier <T> fileCopier,
            IFileExistenceChecker <T> fileExistenceChecker,
            ICopyRequester copyRequester,
            IPathTransformer <T> pathTransformer,
            IContentLocationStore contentLocationStore)
        {
            Contract.Requires(settings != null);
            Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);

            _settings                   = settings;
            _tempFolderForCopies        = new DisposableDirectory(fileSystem, workingDirectory / "Temp");
            _remoteFileCopier           = fileCopier;
            _remoteFileExistenceChecker = fileExistenceChecker;
            _copyRequester              = copyRequester;
            _contentLocationStore       = contentLocationStore;
            _pathTransformer            = pathTransformer;
            _fileSystem                 = fileSystem;

            _workingDirectory = _tempFolderForCopies.Path;

            _ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations);
            _proactiveCopyIoGate       = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations);
            _retryIntervals            = settings.RetryIntervalForCopies;
            _maxRetryCount             = settings.MaxRetryCount;
            _timeoutForPoractiveCopies = settings.TimeoutForProactiveCopies;
        }
Пример #4
0
        /// <nodoc />
        public DistributedContentCopier(
            AbsolutePath workingDirectory,
            DistributedContentStoreSettings settings,
            IAbsFileSystem fileSystem,
            IFileCopier <T> fileCopier,
            IFileExistenceChecker <T> fileExistenceChecker,
            IPathTransformer <T> pathTransformer,
            IContentLocationStore contentLocationStore)
        {
            Contract.Requires(settings != null);
            Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);

            _settings                   = settings;
            _tempFolderForCopies        = new DisposableDirectory(fileSystem, workingDirectory / "Temp");
            _remoteFileCopier           = fileCopier;
            _remoteFileExistenceChecker = fileExistenceChecker;
            _contentLocationStore       = contentLocationStore;
            _pathTransformer            = pathTransformer;
            _fileSystem                 = fileSystem;

            _workingDirectory = _tempFolderForCopies.Path;

            // TODO: Use hashers from IContentStoreInternal instead?
            _hashers = HashInfoLookup.CreateAll();

            _ioGate         = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations);
            _retryIntervals = settings.RetryIntervalForCopies;
        }
Пример #5
0
        /// <nodoc />
        public DistributedContentStore(
            MachineLocation localMachineLocation,
            AbsolutePath localCacheRoot,
            Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
            IContentLocationStoreFactory contentLocationStoreFactory,
            DistributedContentStoreSettings settings,
            DistributedContentCopier <T> distributedCopier,
            IClock clock = null,
            ContentStoreSettings contentStoreSettings = null)
        {
            Contract.Requires(settings != null);

            LocalMachineLocation         = localMachineLocation;
            _contentLocationStoreFactory = contentLocationStoreFactory;
            _clock                  = clock;
            _distributedCopier      = distributedCopier;
            _copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp");

            contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
            _settings            = settings;

            // Queue is created in unstarted state because the eviction function
            // requires the context passed at startup.
            _evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
                Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
                Redis.RedisContentLocationStoreConstants.BatchInterval,
                _settings.LocationStoreBatchSize);

            _enableDistributedEviction = _settings.ReplicaCreditInMinutes != null;
            var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(_settings.ReplicaCreditInMinutes, _settings.LocationStoreBatchSize) : null;

            var enableTouch = _settings.ContentHashBumpTime.HasValue;

            if (enableTouch)
            {
                _contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, _settings.ContentHashBumpTime.Value, clock: _clock);
            }

            TrimBulkAsync trimBulkAsync = null;

            InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);

            if (settings.PinConfiguration?.IsPinCachingEnabled == true)
            {
                _pinCache = new PinCache(clock: _clock);
            }
        }
Пример #6
0
        public DistributedContentStore(
            MachineLocation localMachineLocation,
            AbsolutePath localCacheRoot,
            Func <IDistributedLocationStore, IContentStore> innerContentStoreFunc,
            IContentLocationStoreFactory contentLocationStoreFactory,
            DistributedContentStoreSettings settings,
            DistributedContentCopier distributedCopier,
            IClock?clock = null)
        {
            Contract.Requires(settings != null);

            LocalMachineLocation         = localMachineLocation;
            _contentLocationStoreFactory = contentLocationStoreFactory;
            _clock                  = clock ?? SystemClock.Instance;
            _distributedCopier      = distributedCopier;
            _copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp");

            _settings = settings;

            InnerContentStore = innerContentStoreFunc(this);
        }
Пример #7
0
        public DistributedContentCopier(
            DistributedContentStoreSettings settings,
            IAbsFileSystem fileSystem,
            IRemoteFileCopier fileCopier,
            IContentCommunicationManager copyRequester,
            IClock clock,
            ILogger logger)
        {
            Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);

            _settings            = settings;
            _remoteFileCopier    = fileCopier;
            CommunicationManager = copyRequester;
            FileSystem           = fileSystem;
            _clock = clock;

            Context        = new Context(logger);
            _copyScheduler = settings.CopyScheduler.Create(Context);

            _retryIntervals = settings.RetryIntervalForCopies;
            _maxRetryCount  = settings.MaxRetryCount;
        }
        /// <nodoc />
        public DistributedContentStore(
            MachineLocation localMachineLocation,
            AbsolutePath localCacheRoot,
            Func <ContentStoreSettings, IDistributedLocationStore, IContentStore> innerContentStoreFunc,
            IContentLocationStoreFactory contentLocationStoreFactory,
            DistributedContentStoreSettings settings,
            DistributedContentCopier <T> distributedCopier,
            IClock clock = null,
            ContentStoreSettings contentStoreSettings = null)
        {
            Contract.Requires(settings != null);

            LocalMachineLocation         = localMachineLocation;
            _contentLocationStoreFactory = contentLocationStoreFactory;
            _clock                  = clock;
            _distributedCopier      = distributedCopier;
            _copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp");

            contentStoreSettings ??= ContentStoreSettings.DefaultSettings;
            _settings = settings;

            InnerContentStore = innerContentStoreFunc(contentStoreSettings, this);
        }
        public DistributedContentCopier(
            DistributedContentStoreSettings settings,
            IAbsFileSystem fileSystem,
            IRemoteFileCopier fileCopier,
            IContentCommunicationManager copyRequester,
            IClock clock,
            ILogger logger)
        {
            Contract.Requires(settings != null);
            Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);

            _settings         = settings;
            _remoteFileCopier = fileCopier;
            _copyRequester    = copyRequester;
            FileSystem        = fileSystem;
            _clock            = clock;

            _ioGate = new OrderedSemaphore(_settings.MaxConcurrentCopyOperations, _settings.OrderForCopies, new Context(logger));
            _proactiveCopyIoGate             = new OrderedSemaphore(_settings.MaxConcurrentProactiveCopyOperations, _settings.OrderForProactiveCopies, new Context(logger));
            _retryIntervals                  = settings.RetryIntervalForCopies;
            _maxRetryCount                   = settings.MaxRetryCount;
            _ioGateTimeoutForProactiveCopies = settings.ProactiveCopyIOGateTimeout;
        }
Пример #10
0
        /// <nodoc />
        public DistributedContentStore(
            byte[] localMachineLocation,
            Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
            IContentLocationStoreFactory contentLocationStoreFactory,
            IFileExistenceChecker <T> fileExistenceChecker,
            IFileCopier <T> fileCopier,
            IPathTransformer <T> pathTransform,
            ICopyRequester copyRequester,
            ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee,
            AbsolutePath tempFolderForCopies,
            IAbsFileSystem fileSystem,
            int locationStoreBatchSize,
            DistributedContentStoreSettings settings,
            int?replicaCreditInMinutes = null,
            IClock clock = null,
            bool enableRepairHandling    = false,
            TimeSpan?contentHashBumpTime = null,
            ContentStoreSettings contentStoreSettings = null)
        {
            Contract.Requires(settings != null);

            LocalMachineLocation          = new MachineLocation(localMachineLocation);
            _enableRepairHandling         = enableRepairHandling;
            _contentLocationStoreFactory  = contentLocationStoreFactory;
            _contentAvailabilityGuarantee = contentAvailabilityGuarantee;
            _locationStoreBatchSize       = locationStoreBatchSize;

            contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
            _settings            = settings;

            // Queue is created in unstarted state because the eviction function
            // requires the context passed at startup.
            _evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
                Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
                Redis.RedisContentLocationStoreConstants.BatchInterval,
                _locationStoreBatchSize);

            _distributedCopierFactory = (contentLocationStore) =>
            {
                return(new DistributedContentCopier <T>(
                           tempFolderForCopies,
                           _settings,
                           fileSystem,
                           fileCopier,
                           fileExistenceChecker,
                           copyRequester,
                           pathTransform,
                           contentLocationStore));
            };

            _enableDistributedEviction = replicaCreditInMinutes != null;
            var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(replicaCreditInMinutes, locationStoreBatchSize) : null;

            var enableTouch = contentHashBumpTime.HasValue;

            if (enableTouch)
            {
                _contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, contentHashBumpTime.Value, clock: clock);
            }

            TrimBulkAsync trimBulkAsync = null;

            InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);

            if (settings.PinConfiguration?.UsePinCache == true)
            {
                _pinCache = new PinCache(clock: clock);
            }
        }