public DistributedContentCopier( DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IFileCopier <T> fileCopier, IFileExistenceChecker <T> fileExistenceChecker, IContentCommunicationManager copyRequester, IPathTransformer <T> pathTransformer, IClock clock) { Contract.Requires(settings != null); Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _remoteFileCopier = fileCopier; _remoteFileExistenceChecker = fileExistenceChecker; _copyRequester = copyRequester; _pathTransformer = pathTransformer; FileSystem = fileSystem; _clock = clock; _ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations); _proactiveCopyIoGate = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations); _retryIntervals = settings.RetryIntervalForCopies; _maxRetryCount = settings.MaxRetryCount; _timeoutForProactiveCopies = settings.TimeoutForProactiveCopies; }
public DistributedContentCopier( DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IRemoteFileCopier fileCopier, IContentCommunicationManager copyRequester, IClock clock, ILogger logger) { Contract.Requires(settings != null); Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _remoteFileCopier = fileCopier; _copyRequester = copyRequester; FileSystem = fileSystem; _clock = clock; var context = new Context(logger); _copyScheduler = settings.CopyScheduler.Create(context); if (_copyRequester is StartupShutdownSlimBase slimBase) { slimBase.StartupAsync(context).Result.ThrowIfFailure(); } _retryIntervals = settings.RetryIntervalForCopies; _maxRetryCount = settings.MaxRetryCount; }
internal DistributedCacheServiceArguments CreateDistributedCacheServiceArguments( IAbsolutePathFileCopier copier, IAbsolutePathTransformer pathTransformer, IContentCommunicationManager copyRequester, DistributedContentSettings dcs, HostInfo host, string cacheName, string cacheRootPath, uint grpcPort, int maxSizeQuotaMB, string dataRootPath, CancellationToken ct, int?bufferSizeForGrpcCopies, int?gzipBarrierSizeForGrpcCopies) { var distributedCacheServiceHost = new EnvironmentVariableHost(); var localCasSettings = LocalCasSettings.Default( maxSizeQuotaMB: maxSizeQuotaMB, cacheRootPath: cacheRootPath, cacheName: cacheName, grpcPort: grpcPort, grpcPortFileName: _scenario); localCasSettings.PreferredCacheDrive = Path.GetPathRoot(cacheRootPath); localCasSettings.ServiceSettings = new LocalCasServiceSettings(60, scenarioName: _scenario, grpcPort: grpcPort, grpcPortFileName: _scenario, bufferSizeForGrpcCopies: bufferSizeForGrpcCopies, gzipBarrierSizeForGrpcCopies: gzipBarrierSizeForGrpcCopies, grpcThreadPoolSize: null); var config = new DistributedCacheServiceConfiguration(localCasSettings, dcs); return(new DistributedCacheServiceArguments(_logger, copier, pathTransformer, copyRequester, distributedCacheServiceHost, host, ct, dataRootPath, config, null)); }
internal DistributedCacheServiceArguments CreateDistributedCacheServiceArguments( IRemoteFileCopier copier, IContentCommunicationManager copyRequester, DistributedContentSettings dcs, HostInfo host, string cacheName, string cacheRootPath, uint grpcPort, int maxSizeQuotaMB, string dataRootPath, CancellationToken ct, int?bufferSizeForGrpcCopies, int?gzipBarrierSizeForGrpcCopies, LoggingSettings loggingSettings, ITelemetryFieldsProvider telemetryFieldsProvider) { var distributedCacheServiceHost = new EnvironmentVariableHost(); var localCasSettings = LocalCasSettings.Default( maxSizeQuotaMB: maxSizeQuotaMB, cacheRootPath: cacheRootPath, cacheName: cacheName, grpcPort: grpcPort, grpcPortFileName: _scenario); localCasSettings.PreferredCacheDrive = new AbsolutePath(cacheRootPath).GetPathRoot(); localCasSettings.ServiceSettings = new LocalCasServiceSettings(60, scenarioName: _scenario, grpcPort: grpcPort, grpcPortFileName: _scenario, bufferSizeForGrpcCopies: bufferSizeForGrpcCopies, gzipBarrierSizeForGrpcCopies: gzipBarrierSizeForGrpcCopies); var config = new DistributedCacheServiceConfiguration(localCasSettings, dcs, loggingSettings); return(new DistributedCacheServiceArguments(_logger, copier, copyRequester, distributedCacheServiceHost, host, ct, dataRootPath, config, null) { TelemetryFieldsProvider = telemetryFieldsProvider, }); }
/// <nodoc /> public DistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IFileCopier <T> fileCopier, IFileExistenceChecker <T> fileExistenceChecker, IContentCommunicationManager copyRequester, IPathTransformer <T> pathTransformer, IClock clock, IContentLocationStore contentLocationStore) { Contract.Requires(settings != null); Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _tempFolderForCopies = new DisposableDirectory(fileSystem, workingDirectory / "Temp"); _remoteFileCopier = fileCopier; _remoteFileExistenceChecker = fileExistenceChecker; _copyRequester = copyRequester; _contentLocationStore = contentLocationStore; _pathTransformer = pathTransformer; _fileSystem = fileSystem; _clock = clock; _workingDirectory = _tempFolderForCopies.Path; _ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations); _proactiveCopyIoGate = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations); _retryIntervals = settings.RetryIntervalForCopies; _maxRetryCount = settings.MaxRetryCount; _timeoutForProactiveCopies = settings.TimeoutForProactiveCopies; }
public TestDistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IRemoteFileCopier fileCopier, IContentCommunicationManager copyRequester) : base(settings, fileSystem, fileCopier, copyRequester, TestSystemClock.Instance, TestGlobal.Logger) { Settings = settings; WorkingFolder = workingDirectory; }
public TestDistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IFileCopier <AbsolutePath> fileCopier, IFileExistenceChecker <AbsolutePath> fileExistenceChecker, IContentCommunicationManager copyRequester, IPathTransformer <AbsolutePath> pathTransformer, IContentLocationStore contentLocationStore) : base(workingDirectory, settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransformer, TestSystemClock.Instance, contentLocationStore) { }
public TestDistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IRemoteFileCopier <AbsolutePath> fileCopier, IFileExistenceChecker <AbsolutePath> fileExistenceChecker, IContentCommunicationManager copyRequester, IPathTransformer <AbsolutePath> pathTransformer) : base(settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransformer, TestSystemClock.Instance) { Settings = settings; WorkingFolder = workingDirectory; PathTransformer = pathTransformer as NoOpPathTransformer; }
public DistributedContentCopier( DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IRemoteFileCopier fileCopier, IContentCommunicationManager copyRequester, IClock clock, ILogger logger) { Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _remoteFileCopier = fileCopier; CommunicationManager = copyRequester; FileSystem = fileSystem; _clock = clock; Context = new Context(logger); _copyScheduler = settings.CopyScheduler.Create(Context); _retryIntervals = settings.RetryIntervalForCopies; _maxRetryCount = settings.MaxRetryCount; }
public DistributedContentCopier( DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IRemoteFileCopier fileCopier, IContentCommunicationManager copyRequester, IClock clock, ILogger logger) { Contract.Requires(settings != null); Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _remoteFileCopier = fileCopier; _copyRequester = copyRequester; FileSystem = fileSystem; _clock = clock; _ioGate = new OrderedSemaphore(_settings.MaxConcurrentCopyOperations, _settings.OrderForCopies, new Context(logger)); _proactiveCopyIoGate = new OrderedSemaphore(_settings.MaxConcurrentProactiveCopyOperations, _settings.OrderForProactiveCopies, new Context(logger)); _retryIntervals = settings.RetryIntervalForCopies; _maxRetryCount = settings.MaxRetryCount; _ioGateTimeoutForProactiveCopies = settings.ProactiveCopyIOGateTimeout; }
/// <inheritdoc /> public DistributedCacheServiceArguments( ILogger logger, IAbsolutePathFileCopier copier, IAbsolutePathTransformer pathTransformer, IContentCommunicationManager copyRequester, IDistributedCacheServiceHost host, HostInfo hostInfo, CancellationToken cancellation, string dataRootPath, DistributedCacheServiceConfiguration configuration, string keyspace) { Logger = logger; Copier = copier; CopyRequester = copyRequester; PathTransformer = pathTransformer; Host = host; HostInfo = hostInfo; Cancellation = cancellation; DataRootPath = dataRootPath; Configuration = configuration; Keyspace = ComputeKeySpace(hostInfo, configuration, keyspace); }
/// <nodoc /> public DistributedContentStore( byte[] localMachineLocation, Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, IFileExistenceChecker <T> fileExistenceChecker, IFileCopier <T> fileCopier, IPathTransformer <T> pathTransform, IContentCommunicationManager copyRequester, ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee, AbsolutePath tempFolderForCopies, IAbsFileSystem fileSystem, int locationStoreBatchSize, DistributedContentStoreSettings settings, int?replicaCreditInMinutes = null, IClock clock = null, bool enableRepairHandling = false, TimeSpan?contentHashBumpTime = null, ContentStoreSettings contentStoreSettings = null, bool setPostInitializationCompletionAfterStartup = false) { Contract.Requires(settings != null); _setPostInitializationCompletionAfterStartup = setPostInitializationCompletionAfterStartup; LocalMachineLocation = new MachineLocation(localMachineLocation); _enableRepairHandling = enableRepairHandling; _contentLocationStoreFactory = contentLocationStoreFactory; _contentAvailabilityGuarantee = contentAvailabilityGuarantee; _locationStoreBatchSize = locationStoreBatchSize; _clock = clock; contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings; _settings = settings; // Queue is created in unstarted state because the eviction function // requires the context passed at startup. _evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted( Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism, Redis.RedisContentLocationStoreConstants.BatchInterval, _locationStoreBatchSize); _distributedCopierFactory = (contentLocationStore) => { return(new DistributedContentCopier <T>( tempFolderForCopies, _settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransform, _clock, contentLocationStore)); }; _enableDistributedEviction = replicaCreditInMinutes != null; var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(replicaCreditInMinutes, locationStoreBatchSize) : null; var enableTouch = contentHashBumpTime.HasValue; if (enableTouch) { _contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, contentHashBumpTime.Value, clock: _clock); } TrimBulkAsync trimBulkAsync = null; InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync); if (settings.PinConfiguration?.IsPinCachingEnabled == true) { _pinCache = new PinCache(clock: _clock); } }