public DistributedContentStoreFactory(DistributedCacheServiceArguments arguments) { _logger = arguments.Logger; _arguments = arguments; _distributedSettings = arguments.Configuration.DistributedContentSettings; _keySpace = string.IsNullOrWhiteSpace(_arguments.Keyspace) ? ContentLocationStoreFactory.DefaultKeySpace : _arguments.Keyspace; _fileSystem = new PassThroughFileSystem(_logger); _secretRetriever = new DistributedCacheSecretRetriever(arguments); var bandwidthCheckedCopier = new BandwidthCheckedCopier(_arguments.Copier, BandwidthChecker.Configuration.FromDistributedContentSettings(_distributedSettings)); _orderedResolvedCacheSettings = ResolveCacheSettingsInPrecedenceOrder(arguments); Contract.Assert(_orderedResolvedCacheSettings.Count != 0); RedisContentLocationStoreConfiguration = CreateRedisConfiguration(); _distributedContentStoreSettings = CreateDistributedStoreSettings(_arguments, RedisContentLocationStoreConfiguration); _copier = new DistributedContentCopier <AbsolutePath>( _distributedContentStoreSettings, _fileSystem, fileCopier: bandwidthCheckedCopier, fileExistenceChecker: _arguments.Copier, _arguments.CopyRequester, _arguments.PathTransformer, _arguments.Overrides.Clock ); _redisMemoizationStoreFactory = new Lazy <RedisMemoizationStoreFactory>(() => CreateRedisCacheFactory()); }
private static DistributedContentStoreSettings CreateDistributedStoreSettings( DistributedCacheServiceArguments arguments, RedisContentLocationStoreConfiguration redisContentLocationStoreConfiguration) { var distributedSettings = arguments.Configuration.DistributedContentSettings; PinConfiguration pinConfiguration = new PinConfiguration(); if (distributedSettings.IsPinBetterEnabled) { ApplyIfNotNull(distributedSettings.PinMinUnverifiedCount, v => pinConfiguration.PinMinUnverifiedCount = v); ApplyIfNotNull(distributedSettings.StartCopyWhenPinMinUnverifiedCountThreshold, v => pinConfiguration.StartCopyWhenPinMinUnverifiedCountThreshold = v); ApplyIfNotNull(distributedSettings.MaxIOOperations, v => pinConfiguration.MaxIOOperations = v); } var distributedContentStoreSettings = new DistributedContentStoreSettings() { TrustedHashFileSizeBoundary = distributedSettings.TrustedHashFileSizeBoundary, ParallelHashingFileSizeBoundary = distributedSettings.ParallelHashingFileSizeBoundary, MaxConcurrentCopyOperations = distributedSettings.MaxConcurrentCopyOperations, PinConfiguration = pinConfiguration, RetryIntervalForCopies = distributedSettings.RetryIntervalForCopies, MaxRetryCount = distributedSettings.MaxRetryCount, TimeoutForProactiveCopies = TimeSpan.FromMinutes(distributedSettings.TimeoutForProactiveCopiesMinutes), ProactiveCopyMode = (ProactiveCopyMode)Enum.Parse(typeof(ProactiveCopyMode), distributedSettings.ProactiveCopyMode), PushProactiveCopies = distributedSettings.PushProactiveCopies, ProactiveCopyOnPut = distributedSettings.ProactiveCopyOnPut, ProactiveCopyOnPin = distributedSettings.ProactiveCopyOnPin, ProactiveCopyUsePreferredLocations = distributedSettings.ProactiveCopyUsePreferredLocations, MaxConcurrentProactiveCopyOperations = distributedSettings.MaxConcurrentProactiveCopyOperations, ProactiveCopyLocationsThreshold = distributedSettings.ProactiveCopyLocationsThreshold, ProactiveCopyRejectOldContent = distributedSettings.ProactiveCopyRejectOldContent, ReplicaCreditInMinutes = distributedSettings.IsDistributedEvictionEnabled ? distributedSettings.ReplicaCreditInMinutes : null, EnableRepairHandling = distributedSettings.IsRepairHandlingEnabled, LocationStoreBatchSize = distributedSettings.RedisBatchPageSize, RestrictedCopyReplicaCount = distributedSettings.RestrictedCopyReplicaCount, CopyAttemptsWithRestrictedReplicas = distributedSettings.CopyAttemptsWithRestrictedReplicas, AreBlobsSupported = redisContentLocationStoreConfiguration.AreBlobsSupported, MaxBlobSize = redisContentLocationStoreConfiguration.MaxBlobSize, DelayForProactiveReplication = TimeSpan.FromSeconds(distributedSettings.ProactiveReplicationDelaySeconds), ProactiveReplicationCopyLimit = distributedSettings.ProactiveReplicationCopyLimit, EnableProactiveReplication = distributedSettings.EnableProactiveReplication, TraceProactiveCopy = distributedSettings.TraceProactiveCopy, ProactiveCopyGetBulkBatchSize = distributedSettings.ProactiveCopyGetBulkBatchSize, ProactiveCopyGetBulkInterval = TimeSpan.FromSeconds(distributedSettings.ProactiveCopyGetBulkIntervalSeconds) }; if (distributedSettings.EnableProactiveReplication && redisContentLocationStoreConfiguration.Checkpoint != null) { distributedContentStoreSettings.ProactiveReplicationInterval = redisContentLocationStoreConfiguration.Checkpoint.RestoreCheckpointInterval; } ApplyIfNotNull(distributedSettings.MaximumConcurrentPutAndPlaceFileOperations, v => distributedContentStoreSettings.MaximumConcurrentPutAndPlaceFileOperations = v); arguments.Overrides.Override(distributedContentStoreSettings); ConfigurationPrinter.TraceConfiguration(distributedContentStoreSettings, arguments.Logger); return(distributedContentStoreSettings); }
public DistributedContentStoreFactory(DistributedCacheServiceArguments arguments) { _logger = arguments.Logger; _arguments = arguments; _distributedSettings = arguments.Configuration.DistributedContentSettings; _keySpace = string.IsNullOrWhiteSpace(_arguments.Keyspace) ? ContentLocationStoreFactory.DefaultKeySpace : _arguments.Keyspace; _fileSystem = arguments.FileSystem; _secretRetriever = new DistributedCacheSecretRetriever(arguments); _orderedResolvedCacheSettings = ResolveCacheSettingsInPrecedenceOrder(arguments); Contract.Assert(_orderedResolvedCacheSettings.Count != 0); RedisContentLocationStoreConfiguration = CreateRedisConfiguration(); _distributedContentStoreSettings = CreateDistributedStoreSettings(_arguments, RedisContentLocationStoreConfiguration); _copier = new DistributedContentCopier( _distributedContentStoreSettings, _fileSystem, fileCopier: _arguments.Copier, copyRequester: _arguments.CopyRequester, _arguments.Overrides.Clock, _logger ); _redisMemoizationStoreFactory = new Lazy <RedisMemoizationStoreFactory>(() => CreateRedisCacheFactory()); }
public override void Override(DistributedContentStoreSettings settings) { settings.InlineOperationsForTests = true; settings.SetPostInitializationCompletionAfterStartup = true; _tests._overrideDistributedContentStooreSettings?.Invoke(settings); }
public TestDistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IRemoteFileCopier fileCopier, IContentCommunicationManager copyRequester) : base(settings, fileSystem, fileCopier, copyRequester, TestSystemClock.Instance, TestGlobal.Logger) { Settings = settings; WorkingFolder = workingDirectory; }
public TestDistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IFileCopier <AbsolutePath> fileCopier, IFileExistenceChecker <AbsolutePath> fileExistenceChecker, IContentCommunicationManager copyRequester, IPathTransformer <AbsolutePath> pathTransformer, IContentLocationStore contentLocationStore) : base(workingDirectory, settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransformer, TestSystemClock.Instance, contentLocationStore) { }
public TestDistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IRemoteFileCopier <AbsolutePath> fileCopier, IFileExistenceChecker <AbsolutePath> fileExistenceChecker, IContentCommunicationManager copyRequester, IPathTransformer <AbsolutePath> pathTransformer) : base(settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransformer, TestSystemClock.Instance) { Settings = settings; WorkingFolder = workingDirectory; PathTransformer = pathTransformer as NoOpPathTransformer; }
/// <nodoc /> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier contentCopier, IDistributedContentCopierHost copierHost, MachineLocation localMachineLocation, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentCopier, copierHost, localMachineLocation, settings) { }
/// <nodoc /> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier contentCopier, DistributedContentStore distributedStore, MachineLocation localMachineLocation, ColdStorage coldStorage, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentCopier, distributedStore, localMachineLocation, coldStorage, settings) { }
/// <summary> /// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class. /// </summary> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier <T> contentCopier, MachineLocation localMachineLocation, PinCache pinCache = null, ContentTrackerUpdater contentTrackerUpdater = null, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentCopier, localMachineLocation, pinCache: pinCache, contentTrackerUpdater: contentTrackerUpdater, settings) { _putFileGate = new SemaphoreSlim(settings.MaximumConcurrentPutFileOperations); }
/// <summary> /// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class. /// </summary> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, ContentAvailabilityGuarantee contentAvailabilityGuarantee, DistributedContentCopier <T> contentCopier, byte[] localMachineLocation, PinCache pinCache = null, ContentTrackerUpdater contentTrackerUpdater = null, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentAvailabilityGuarantee, contentCopier, localMachineLocation, pinCache: pinCache, contentTrackerUpdater: contentTrackerUpdater, settings) { }
/// <summary> /// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class. /// </summary> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier <T> contentCopier, IDistributedContentCopierHost copierHost, MachineLocation localMachineLocation, PinCache pinCache = null, ContentTrackerUpdater contentTrackerUpdater = null, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentCopier, copierHost, localMachineLocation, pinCache: pinCache, contentTrackerUpdater: contentTrackerUpdater, settings) { }
public virtual void Override(DistributedContentStoreSettings settings) { }
private DistributedContentStoreSettings Override(DistributedContentStoreSettings settings) { _arguments.Overrides.Override(settings); return(settings); }
private static DistributedContentStoreSettings CreateDistributedStoreSettings( DistributedCacheServiceArguments arguments, RedisContentLocationStoreConfiguration redisContentLocationStoreConfiguration) { var distributedSettings = arguments.Configuration.DistributedContentSettings; ContentAvailabilityGuarantee contentAvailabilityGuarantee; if (string.IsNullOrEmpty(distributedSettings.ContentAvailabilityGuarantee)) { contentAvailabilityGuarantee = ContentAvailabilityGuarantee.FileRecordsExist; } else if (!Enum.TryParse(distributedSettings.ContentAvailabilityGuarantee, true, out contentAvailabilityGuarantee)) { throw new ArgumentException($"Unable to parse {nameof(distributedSettings.ContentAvailabilityGuarantee)}: [{distributedSettings.ContentAvailabilityGuarantee}]"); } PinConfiguration pinConfiguration = null; if (distributedSettings.IsPinBetterEnabled) { pinConfiguration = new PinConfiguration(); ApplyIfNotNull(distributedSettings.PinRisk, v => pinConfiguration.PinRisk = v); ApplyIfNotNull(distributedSettings.PinMinUnverifiedCount, v => pinConfiguration.PinMinUnverifiedCount = v); ApplyIfNotNull(distributedSettings.MachineRisk, v => pinConfiguration.MachineRisk = v); ApplyIfNotNull(distributedSettings.FileRisk, v => pinConfiguration.FileRisk = v); ApplyIfNotNull(distributedSettings.MaxIOOperations, v => pinConfiguration.MaxIOOperations = v); pinConfiguration.IsPinCachingEnabled = distributedSettings.IsPinCachingEnabled; ApplyIfNotNull(distributedSettings.PinCacheReplicaCreditRetentionMinutes, v => pinConfiguration.PinCachePerReplicaRetentionCreditMinutes = v); ApplyIfNotNull(distributedSettings.PinCacheReplicaCreditRetentionDecay, v => pinConfiguration.PinCacheReplicaCreditRetentionFactor = v); } var contentHashBumpTime = TimeSpan.FromMinutes(distributedSettings.ContentHashBumpTimeMinutes); var lazyTouchContentHashBumpTime = distributedSettings.IsTouchEnabled ? (TimeSpan?)contentHashBumpTime : null; if (redisContentLocationStoreConfiguration.ReadMode == ContentLocationMode.LocalLocationStore) { // LocalLocationStore has its own internal notion of lazy touch/registration. We disable the lazy touch in distributed content store // because it can conflict with behavior of the local location store. lazyTouchContentHashBumpTime = null; } var distributedContentStoreSettings = new DistributedContentStoreSettings() { TrustedHashFileSizeBoundary = distributedSettings.TrustedHashFileSizeBoundary, ParallelHashingFileSizeBoundary = distributedSettings.ParallelHashingFileSizeBoundary, MaxConcurrentCopyOperations = distributedSettings.MaxConcurrentCopyOperations, PinConfiguration = pinConfiguration, RetryIntervalForCopies = distributedSettings.RetryIntervalForCopies, MaxRetryCount = distributedSettings.MaxRetryCount, TimeoutForProactiveCopies = TimeSpan.FromMinutes(distributedSettings.TimeoutForProactiveCopiesMinutes), ProactiveCopyMode = (ProactiveCopyMode)Enum.Parse(typeof(ProactiveCopyMode), distributedSettings.ProactiveCopyMode), PushProactiveCopies = distributedSettings.PushProactiveCopies, ProactiveCopyOnPut = distributedSettings.ProactiveCopyOnPut, ProactiveCopyOnPin = distributedSettings.ProactiveCopyOnPin, ProactiveCopyUsePreferredLocations = distributedSettings.ProactiveCopyUsePreferredLocations, MaxConcurrentProactiveCopyOperations = distributedSettings.MaxConcurrentProactiveCopyOperations, ProactiveCopyLocationsThreshold = distributedSettings.ProactiveCopyLocationsThreshold, ProactiveCopyRejectOldContent = distributedSettings.ProactiveCopyRejectOldContent, ReplicaCreditInMinutes = distributedSettings.IsDistributedEvictionEnabled ? distributedSettings.ReplicaCreditInMinutes : null, EnableRepairHandling = distributedSettings.IsRepairHandlingEnabled, ContentHashBumpTime = lazyTouchContentHashBumpTime, LocationStoreBatchSize = distributedSettings.RedisBatchPageSize, ContentAvailabilityGuarantee = contentAvailabilityGuarantee, PrioritizeDesignatedLocationsOnCopies = distributedSettings.PrioritizeDesignatedLocationsOnCopies, RestrictedCopyReplicaCount = distributedSettings.RestrictedCopyReplicaCount, CopyAttemptsWithRestrictedReplicas = distributedSettings.CopyAttemptsWithRestrictedReplicas, AreBlobsSupported = redisContentLocationStoreConfiguration.AreBlobsSupported, MaxBlobSize = redisContentLocationStoreConfiguration.MaxBlobSize, DelayForProactiveReplication = TimeSpan.FromSeconds(distributedSettings.ProactiveReplicationDelaySeconds), ProactiveReplicationCopyLimit = distributedSettings.ProactiveReplicationCopyLimit, EnableProactiveReplication = distributedSettings.EnableProactiveReplication, TraceProactiveCopy = distributedSettings.TraceProactiveCopy, }; if (distributedSettings.EnableProactiveReplication && redisContentLocationStoreConfiguration.Checkpoint != null) { distributedContentStoreSettings.ProactiveReplicationInterval = redisContentLocationStoreConfiguration.Checkpoint.RestoreCheckpointInterval; } ApplyIfNotNull(distributedSettings.MaximumConcurrentPutAndPlaceFileOperations, v => distributedContentStoreSettings.MaximumConcurrentPutAndPlaceFileOperations = v); arguments.Overrides.Override(distributedContentStoreSettings); ConfigurationPrinter.TraceConfiguration(distributedContentStoreSettings, arguments.Logger); return(distributedContentStoreSettings); }