public DistributedContentStoreFactory(DistributedCacheServiceArguments arguments)
{
_logger = arguments.Logger;
_arguments = arguments;
_distributedSettings = arguments.Configuration.DistributedContentSettings;
_keySpace = string.IsNullOrWhiteSpace(_arguments.Keyspace) ? ContentLocationStoreFactory.DefaultKeySpace : _arguments.Keyspace;
_fileSystem = arguments.FileSystem;
_secretRetriever = new DistributedCacheSecretRetriever(arguments);
_orderedResolvedCacheSettings = ResolveCacheSettingsInPrecedenceOrder(arguments);
Contract.Assert(_orderedResolvedCacheSettings.Count != 0);
RedisContentLocationStoreConfiguration = CreateRedisConfiguration();
_distributedContentStoreSettings = CreateDistributedStoreSettings(_arguments, RedisContentLocationStoreConfiguration);
_copier = new DistributedContentCopier(
_distributedContentStoreSettings,
_fileSystem,
fileCopier: _arguments.Copier,
copyRequester: _arguments.CopyRequester,
_arguments.Overrides.Clock,
_logger
);
_redisMemoizationStoreFactory = new Lazy <RedisMemoizationStoreFactory>(() => CreateRedisCacheFactory());
}
public DistributedContentStoreFactory(DistributedCacheServiceArguments arguments)
{
_logger = arguments.Logger;
_arguments = arguments;
_distributedSettings = arguments.Configuration.DistributedContentSettings;
_keySpace = string.IsNullOrWhiteSpace(_arguments.Keyspace) ? ContentLocationStoreFactory.DefaultKeySpace : _arguments.Keyspace;
_fileSystem = new PassThroughFileSystem(_logger);
_secretRetriever = new DistributedCacheSecretRetriever(arguments);
var bandwidthCheckedCopier = new BandwidthCheckedCopier(_arguments.Copier, BandwidthChecker.Configuration.FromDistributedContentSettings(_distributedSettings));
_orderedResolvedCacheSettings = ResolveCacheSettingsInPrecedenceOrder(arguments);
Contract.Assert(_orderedResolvedCacheSettings.Count != 0);
RedisContentLocationStoreConfiguration = CreateRedisConfiguration();
_distributedContentStoreSettings = CreateDistributedStoreSettings(_arguments, RedisContentLocationStoreConfiguration);
_copier = new DistributedContentCopier <AbsolutePath>(
_distributedContentStoreSettings,
_fileSystem,
fileCopier: bandwidthCheckedCopier,
fileExistenceChecker: _arguments.Copier,
_arguments.CopyRequester,
_arguments.PathTransformer,
_arguments.Overrides.Clock
);
_redisMemoizationStoreFactory = new Lazy <RedisMemoizationStoreFactory>(() => CreateRedisCacheFactory());
}
public RedisMemoizationStoreFactory CreateRedisCacheFactory(AbsolutePath localCacheRoot, out RedisContentLocationStoreConfiguration config)
{
var redisContentLocationStoreConfiguration = new RedisMemoizationStoreConfiguration
{
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
EvictionWindowSize = _distributedSettings.EvictionWindowSize,
EvictionPoolSize = _distributedSettings.EvictionPoolSize,
EvictionRemovalFraction = _distributedSettings.EvictionRemovalFraction,
MemoizationExpiryTime = TimeSpan.FromMinutes(_distributedSettings.RedisMemoizationExpiryTimeMinutes)
};
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineExpiryMinutes, v => redisContentLocationStoreConfiguration.MachineExpiry = TimeSpan.FromMinutes(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(localCacheRoot / "LocationDb")
{
StoreClusterState = _distributedSettings.StoreClusterStateInDatabase
};
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.GarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheEnabled, v => dbConfig.ContentCacheEnabled = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushDegreeOfParallelism, v => dbConfig.FlushDegreeOfParallelism = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushSingleTransaction, v => dbConfig.FlushSingleTransaction = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushPreservePercentInMemory, v => dbConfig.FlushPreservePercentInMemory = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheMaximumUpdatesPerFlush, v => dbConfig.CacheMaximumUpdatesPerFlush = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheFlushingMaximumInterval, v => dbConfig.CacheFlushingMaximumInterval = v);
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionIntervalMinutes,
v => dbConfig.FullRangeCompactionInterval = TimeSpan.FromMinutes(v));
redisContentLocationStoreConfiguration.Database = dbConfig;
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, localCacheRoot).GetAwaiter().GetResult();
}
if (_distributedSettings.IsRedisGarbageCollectionEnabled)
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = new RedisGarbageCollectionConfiguration()
{
MaximumEntryLastAccessTime = TimeSpan.FromMinutes(30)
};
}
else
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = null;
}
var contentHashBumpTime = TimeSpan.FromMinutes(_distributedSettings.ContentHashBumpTimeMinutes);
// RedisContentSecretName and RedisMachineLocationsSecretName can be null. HostConnectionStringProvider won't fail in this case.
IConnectionStringProvider contentConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisContentSecretName);
IConnectionStringProvider machineLocationsConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisMachineLocationsSecretName);
config = redisContentLocationStoreConfiguration;
return(new RedisMemoizationStoreFactory(
contentConnectionStringProvider,
machineLocationsConnectionStringProvider,
SystemClock.Instance,
contentHashBumpTime,
_keySpace,
configuration: redisContentLocationStoreConfiguration
));
}
private RedisMemoizationStoreConfiguration CreateRedisConfiguration()
{
var primaryCacheRoot = OrderedResolvedCacheSettings[0].ResolvedCacheRootPath;
var redisContentLocationStoreConfiguration = new RedisMemoizationStoreConfiguration
{
Keyspace = _keySpace + RedisKeySpaceSalt,
LogReconciliationHashes = _distributedSettings.LogReconciliationHashes,
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
UseFullEvictionSort = _distributedSettings.UseFullEvictionSort,
EvictionWindowSize = _distributedSettings.EvictionWindowSize,
EvictionPoolSize = _distributedSettings.EvictionPoolSize,
UpdateStaleLocalLastAccessTimes = _distributedSettings.UpdateStaleLocalLastAccessTimes,
EvictionRemovalFraction = _distributedSettings.EvictionRemovalFraction,
EvictionDiscardFraction = _distributedSettings.EvictionDiscardFraction,
UseTieredDistributedEviction = _distributedSettings.UseTieredDistributedEviction,
MemoizationExpiryTime = TimeSpan.FromMinutes(_distributedSettings.RedisMemoizationExpiryTimeMinutes),
ProactiveCopyLocationsThreshold = _distributedSettings.ProactiveCopyLocationsThreshold,
UseBinManager = _distributedSettings.UseBinManager || _distributedSettings.ProactiveCopyUsePreferredLocations,
PreferredLocationsExpiryTime = TimeSpan.FromMinutes(_distributedSettings.PreferredLocationsExpiryTimeMinutes),
PrimaryMachineLocation = OrderedResolvedCacheSettings[0].MachineLocation,
AdditionalMachineLocations = OrderedResolvedCacheSettings.Skip(1).Select(r => r.MachineLocation).ToArray(),
MachineListPrioritizeDesignatedLocations = _distributedSettings.PrioritizeDesignatedLocationsOnCopies,
MachineListDeprioritizeMaster = _distributedSettings.DeprioritizeMasterOnCopies,
TouchContentHashLists = _distributedSettings.TouchContentHashLists
};
ApplyIfNotNull(_distributedSettings.ThrottledEvictionIntervalMinutes, v => redisContentLocationStoreConfiguration.ThrottledEvictionInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.RedisConnectionErrorLimit, v => redisContentLocationStoreConfiguration.RedisConnectionErrorLimit = v);
ApplyIfNotNull(_distributedSettings.RedisReconnectionLimitBeforeServiceRestart, v => redisContentLocationStoreConfiguration.RedisReconnectionLimitBeforeServiceRestart = v);
ApplyIfNotNull(_distributedSettings.TraceRedisFailures, v => redisContentLocationStoreConfiguration.TraceRedisFailures = v);
ApplyIfNotNull(_distributedSettings.TraceRedisTransientFailures, v => redisContentLocationStoreConfiguration.TraceRedisTransientFailures = v);
ApplyIfNotNull(_distributedSettings.MinRedisReconnectInterval, v => redisContentLocationStoreConfiguration.MinRedisReconnectInterval = v);
ApplyIfNotNull(_distributedSettings.CancelBatchWhenMultiplexerIsClosed, v => redisContentLocationStoreConfiguration.CancelBatchWhenMultiplexerIsClosed = v);
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineStateRecomputeIntervalMinutes, v => redisContentLocationStoreConfiguration.MachineStateRecomputeInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineActiveToClosedIntervalMinutes, v => redisContentLocationStoreConfiguration.MachineActiveToClosedInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineActiveToExpiredIntervalMinutes, v => redisContentLocationStoreConfiguration.MachineActiveToExpiredInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.TouchFrequencyMinutes, v => redisContentLocationStoreConfiguration.TouchFrequency = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.EvictionMinAgeMinutes, v => redisContentLocationStoreConfiguration.EvictionMinAge = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.RetryWindowSeconds, v => redisContentLocationStoreConfiguration.RetryWindow = TimeSpan.FromSeconds(v));
ApplyIfNotNull(_distributedSettings.RedisGetBlobTimeoutMilliseconds, v => redisContentLocationStoreConfiguration.BlobTimeout = TimeSpan.FromMilliseconds(v));
ApplyIfNotNull(_distributedSettings.RedisGetCheckpointStateTimeoutInSeconds, v => redisContentLocationStoreConfiguration.ClusterRedisOperationTimeout = TimeSpan.FromSeconds(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(primaryCacheRoot / "LocationDb")
{
LogsKeepLongTerm = true,
UseContextualEntryOperationLogging = _distributedSettings.UseContextualEntryDatabaseOperationLogging,
TraceTouches = _distributedSettings.TraceTouches,
};
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.GarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
if (_distributedSettings.EnableDistributedCache)
{
dbConfig.MetadataGarbageCollectionEnabled = true;
dbConfig.MetadataGarbageCollectionMaximumNumberOfEntriesToKeep = _distributedSettings.MaximumNumberOfMetadataEntriesToStore;
}
ApplyIfNotNull(
_distributedSettings.ContentLocationDatabaseOpenReadOnly,
v => dbConfig.OpenReadOnly = v && !_distributedSettings.IsMasterEligible);
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionIntervalMinutes,
v =>
{
if (v < 0)
{
throw new ArgumentException($"`{nameof(_distributedSettings.FullRangeCompactionIntervalMinutes)}` must be greater than or equal to 0");
}
if (v > 0)
{
dbConfig.FullRangeCompactionInterval = TimeSpan.FromMinutes(v);
}
});
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionVariant,
v =>
{
if (!Enum.TryParse <FullRangeCompactionVariant>(v, out var variant))
{
throw new ArgumentException($"Failed to parse `{nameof(_distributedSettings.FullRangeCompactionVariant)}` setting with value `{_distributedSettings.FullRangeCompactionVariant}` into type `{nameof(FullRangeCompactionVariant)}`");
}
dbConfig.FullRangeCompactionVariant = variant;
});
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionByteIncrementStep,
v => dbConfig.FullRangeCompactionByteIncrementStep = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseEnableDynamicLevelTargetSizes, v => dbConfig.EnableDynamicLevelTargetSizes = v);
if (_distributedSettings.ContentLocationDatabaseLogsBackupEnabled)
{
dbConfig.LogsBackupPath = primaryCacheRoot / "LocationDbLogs";
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseLogsBackupRetentionMinutes, v => dbConfig.LogsRetention = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseEnumerateSortedKeysFromStorageBufferSize, v => dbConfig.EnumerateSortedKeysFromStorageBufferSize = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseEnumerateEntriesWithSortedKeysFromStorageBufferSize, v => dbConfig.EnumerateEntriesWithSortedKeysFromStorageBufferSize = v);
redisContentLocationStoreConfiguration.Database = dbConfig;
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, primaryCacheRoot, dbConfig).GetAwaiter().GetResult();
}
_arguments.Overrides.Override(redisContentLocationStoreConfiguration);
ConfigurationPrinter.TraceConfiguration(redisContentLocationStoreConfiguration, _logger);
return(redisContentLocationStoreConfiguration);
}
private static DistributedContentStoreSettings CreateDistributedStoreSettings(
DistributedCacheServiceArguments arguments,
RedisMemoizationStoreConfiguration redisContentLocationStoreConfiguration)
{
var distributedSettings = arguments.Configuration.DistributedContentSettings;
PinConfiguration pinConfiguration = new PinConfiguration();
if (distributedSettings.IsPinBetterEnabled)
{
ApplyIfNotNull(distributedSettings.PinMinUnverifiedCount, v => pinConfiguration.PinMinUnverifiedCount = v);
ApplyIfNotNull(distributedSettings.StartCopyWhenPinMinUnverifiedCountThreshold, v => pinConfiguration.AsyncCopyOnPinThreshold = v);
ApplyIfNotNull(distributedSettings.AsyncCopyOnPinThreshold, v => pinConfiguration.AsyncCopyOnPinThreshold = v);
ApplyIfNotNull(distributedSettings.MaxIOOperations, v => pinConfiguration.MaxIOOperations = v);
}
var distributedContentStoreSettings = new DistributedContentStoreSettings()
{
TrustedHashFileSizeBoundary = distributedSettings.TrustedHashFileSizeBoundary,
ParallelHashingFileSizeBoundary = distributedSettings.ParallelHashingFileSizeBoundary,
PinConfiguration = pinConfiguration,
RetryIntervalForCopies = distributedSettings.RetryIntervalForCopies,
BandwidthConfigurations = FromDistributedSettings(distributedSettings.BandwidthConfigurations),
MaxRetryCount = distributedSettings.MaxRetryCount,
ProactiveCopyMode = (ProactiveCopyMode)Enum.Parse(typeof(ProactiveCopyMode), distributedSettings.ProactiveCopyMode),
PushProactiveCopies = distributedSettings.PushProactiveCopies,
ProactiveCopyOnPut = distributedSettings.ProactiveCopyOnPut,
ProactiveCopyOnPin = distributedSettings.ProactiveCopyOnPin,
ProactiveCopyUsePreferredLocations = distributedSettings.ProactiveCopyUsePreferredLocations,
ProactiveCopyLocationsThreshold = distributedSettings.ProactiveCopyLocationsThreshold,
ProactiveCopyRejectOldContent = distributedSettings.ProactiveCopyRejectOldContent,
ReplicaCreditInMinutes = distributedSettings.IsDistributedEvictionEnabled ? distributedSettings.ReplicaCreditInMinutes : null,
EnableRepairHandling = distributedSettings.IsRepairHandlingEnabled,
LocationStoreBatchSize = distributedSettings.RedisBatchPageSize,
RestrictedCopyReplicaCount = distributedSettings.RestrictedCopyReplicaCount,
CopyAttemptsWithRestrictedReplicas = distributedSettings.CopyAttemptsWithRestrictedReplicas,
PeriodicCopyTracingInterval = TimeSpan.FromMinutes(distributedSettings.PeriodicCopyTracingIntervalMinutes),
AreBlobsSupported = redisContentLocationStoreConfiguration.AreBlobsSupported,
MaxBlobSize = redisContentLocationStoreConfiguration.MaxBlobSize,
DelayForProactiveReplication = TimeSpan.FromSeconds(distributedSettings.ProactiveReplicationDelaySeconds),
ProactiveReplicationCopyLimit = distributedSettings.ProactiveReplicationCopyLimit,
EnableProactiveReplication = distributedSettings.EnableProactiveReplication,
TraceProactiveCopy = distributedSettings.TraceProactiveCopy,
ProactiveCopyGetBulkBatchSize = distributedSettings.ProactiveCopyGetBulkBatchSize,
ProactiveCopyGetBulkInterval = TimeSpan.FromSeconds(distributedSettings.ProactiveCopyGetBulkIntervalSeconds),
ProactiveCopyMaxRetries = distributedSettings.ProactiveCopyMaxRetries,
};
ApplyIfNotNull(distributedSettings.GrpcCopyCompressionSizeThreshold, v => distributedContentStoreSettings.GrpcCopyCompressionSizeThreshold = v);
ApplyEnumIfNotNull <CopyCompression>(distributedSettings.GrpcCopyCompressionAlgorithm, v => distributedContentStoreSettings.GrpcCopyCompressionAlgorithm = v);
if (distributedSettings.EnableProactiveReplication && redisContentLocationStoreConfiguration.Checkpoint != null)
{
distributedContentStoreSettings.ProactiveReplicationInterval = redisContentLocationStoreConfiguration.Checkpoint.RestoreCheckpointInterval;
ApplyIfNotNull(
distributedSettings.ProactiveReplicationIntervalMinutes,
value => distributedContentStoreSettings.ProactiveReplicationInterval = TimeSpan.FromMinutes(value));
}
ApplyIfNotNull(distributedSettings.MaximumConcurrentPutAndPlaceFileOperations, v => distributedContentStoreSettings.MaximumConcurrentPutAndPlaceFileOperations = v);
distributedContentStoreSettings.CopyScheduler = CopySchedulerConfiguration.FromDistributedContentSettings(distributedSettings);
arguments.Overrides.Override(distributedContentStoreSettings);
ConfigurationPrinter.TraceConfiguration(distributedContentStoreSettings, arguments.Logger);
return(distributedContentStoreSettings);
}
private RedisMemoizationStoreConfiguration CreateRedisConfiguration()
{
var primaryCacheRoot = OrderedResolvedCacheSettings[0].ResolvedCacheRootPath;
var redisContentLocationStoreConfiguration = new RedisMemoizationStoreConfiguration
{
Keyspace = _keySpace + RedisKeySpaceSalt,
LogReconciliationHashes = _distributedSettings.LogReconciliationHashes,
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
UseFullEvictionSort = _distributedSettings.UseFullEvictionSort,
EvictionWindowSize = _distributedSettings.EvictionWindowSize,
EvictionPoolSize = _distributedSettings.EvictionPoolSize,
UpdateStaleLocalLastAccessTimes = _distributedSettings.UpdateStaleLocalLastAccessTimes,
EvictionRemovalFraction = _distributedSettings.EvictionRemovalFraction,
EvictionDiscardFraction = _distributedSettings.EvictionDiscardFraction,
UseTieredDistributedEviction = _distributedSettings.UseTieredDistributedEviction,
MemoizationExpiryTime = TimeSpan.FromMinutes(_distributedSettings.RedisMemoizationExpiryTimeMinutes),
ProactiveCopyLocationsThreshold = _distributedSettings.ProactiveCopyLocationsThreshold,
UseBinManager = _distributedSettings.UseBinManager || _distributedSettings.ProactiveCopyUsePreferredLocations,
PreferredLocationsExpiryTime = TimeSpan.FromMinutes(_distributedSettings.PreferredLocationsExpiryTimeMinutes),
PrimaryMachineLocation = OrderedResolvedCacheSettings[0].MachineLocation,
MachineListPrioritizeDesignatedLocations = _distributedSettings.PrioritizeDesignatedLocationsOnCopies,
MachineListDeprioritizeMaster = _distributedSettings.DeprioritizeMasterOnCopies,
TouchContentHashLists = _distributedSettings.TouchContentHashLists,
};
ApplyIfNotNull(_distributedSettings.BlobOperationLimitCount, v => redisContentLocationStoreConfiguration.BlobOperationLimitCount = v);
ApplyIfNotNull(_distributedSettings.BlobOperationLimitSpanSeconds, v => redisContentLocationStoreConfiguration.BlobOperationLimitSpan = TimeSpan.FromSeconds(v));
ApplyIfNotNull(_distributedSettings.UseSeparateConnectionForRedisBlobs, v => redisContentLocationStoreConfiguration.UseSeparateConnectionForRedisBlobs = v);
// Stop passing additional stores when fully transitioned to unified mode
// since all drives appear under the same machine id
if (_distributedSettings.GetMultiplexMode() != MultiplexMode.Unified)
{
redisContentLocationStoreConfiguration.AdditionalMachineLocations = OrderedResolvedCacheSettings.Skip(1).Select(r => r.MachineLocation).ToArray();
}
ApplyIfNotNull(_distributedSettings.ThrottledEvictionIntervalMinutes, v => redisContentLocationStoreConfiguration.ThrottledEvictionInterval = TimeSpan.FromMinutes(v));
// Redis-related configuration.
ApplyIfNotNull(_distributedSettings.RedisConnectionErrorLimit, v => redisContentLocationStoreConfiguration.RedisConnectionErrorLimit = v);
ApplyIfNotNull(_distributedSettings.RedisReconnectionLimitBeforeServiceRestart, v => redisContentLocationStoreConfiguration.RedisReconnectionLimitBeforeServiceRestart = v);
ApplyIfNotNull(_distributedSettings.DefaultRedisOperationTimeoutInSeconds, v => redisContentLocationStoreConfiguration.OperationTimeout = TimeSpan.FromSeconds(v));
ApplyIfNotNull(_distributedSettings.MinRedisReconnectInterval, v => redisContentLocationStoreConfiguration.MinRedisReconnectInterval = v);
ApplyIfNotNull(_distributedSettings.CancelBatchWhenMultiplexerIsClosed, v => redisContentLocationStoreConfiguration.CancelBatchWhenMultiplexerIsClosed = v);
if (_distributedSettings.RedisExponentialBackoffRetryCount != null)
{
var exponentialBackoffConfiguration = new ExponentialBackoffConfiguration(
_distributedSettings.RedisExponentialBackoffRetryCount.Value,
minBackoff: IfNotNull(_distributedSettings.RedisExponentialBackoffMinIntervalInSeconds, v => TimeSpan.FromSeconds(v)),
maxBackoff: IfNotNull(_distributedSettings.RedisExponentialBackoffMaxIntervalInSeconds, v => TimeSpan.FromSeconds(v)),
deltaBackoff: IfNotNull(_distributedSettings.RedisExponentialBackoffDeltaIntervalInSeconds, v => TimeSpan.FromSeconds(v))
);
redisContentLocationStoreConfiguration.ExponentialBackoffConfiguration = exponentialBackoffConfiguration;
}
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineStateRecomputeIntervalMinutes, v => redisContentLocationStoreConfiguration.MachineStateRecomputeInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineActiveToClosedIntervalMinutes, v => redisContentLocationStoreConfiguration.MachineActiveToClosedInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineActiveToExpiredIntervalMinutes, v => redisContentLocationStoreConfiguration.MachineActiveToExpiredInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.TouchFrequencyMinutes, v => redisContentLocationStoreConfiguration.TouchFrequency = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.ReconcileCacheLifetimeMinutes, v => redisContentLocationStoreConfiguration.ReconcileCacheLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.EvictionMinAgeMinutes, v => redisContentLocationStoreConfiguration.EvictionMinAge = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.RetryWindowSeconds, v => redisContentLocationStoreConfiguration.RetryWindow = TimeSpan.FromSeconds(v));
ApplyIfNotNull(_distributedSettings.Unsafe_MasterThroughputCheckMode, v => redisContentLocationStoreConfiguration.MasterThroughputCheckMode = v);
ApplyIfNotNull(_distributedSettings.Unsafe_EventHubCursorPosition, v => redisContentLocationStoreConfiguration.EventHubCursorPosition = v);
ApplyIfNotNull(_distributedSettings.RedisGetBlobTimeoutMilliseconds, v => redisContentLocationStoreConfiguration.BlobTimeout = TimeSpan.FromMilliseconds(v));
ApplyIfNotNull(_distributedSettings.RedisGetCheckpointStateTimeoutInSeconds, v => redisContentLocationStoreConfiguration.ClusterRedisOperationTimeout = TimeSpan.FromSeconds(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = RocksDbContentLocationDatabaseConfiguration.FromDistributedContentSettings(
_distributedSettings,
primaryCacheRoot / "LocationDb",
primaryCacheRoot / "LocationDbLogs",
logsKeepLongTerm: true
);
redisContentLocationStoreConfiguration.Database = dbConfig;
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, primaryCacheRoot, dbConfig).GetAwaiter().GetResult();
}
_arguments.Overrides.Override(redisContentLocationStoreConfiguration);
ConfigurationPrinter.TraceConfiguration(redisContentLocationStoreConfiguration, _logger);
return(redisContentLocationStoreConfiguration);
}
private RedisMemoizationStoreConfiguration CreateRedisConfiguration()
{
var primaryCacheRoot = OrderedResolvedCacheSettings[0].ResolvedCacheRootPath;
var redisContentLocationStoreConfiguration = new RedisMemoizationStoreConfiguration
{
Keyspace = _keySpace + RedisKeySpaceSalt,
LogReconciliationHashes = _distributedSettings.LogReconciliationHashes,
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
UseFullEvictionSort = _distributedSettings.UseFullEvictionSort,
EvictionWindowSize = _distributedSettings.EvictionWindowSize,
EvictionPoolSize = _distributedSettings.EvictionPoolSize,
UpdateStaleLocalLastAccessTimes = _distributedSettings.UpdateStaleLocalLastAccessTimes,
EvictionRemovalFraction = _distributedSettings.EvictionRemovalFraction,
EvictionDiscardFraction = _distributedSettings.EvictionDiscardFraction,
UseTieredDistributedEviction = _distributedSettings.UseTieredDistributedEviction,
MemoizationExpiryTime = TimeSpan.FromMinutes(_distributedSettings.RedisMemoizationExpiryTimeMinutes),
ProactiveCopyLocationsThreshold = _distributedSettings.ProactiveCopyLocationsThreshold,
UseBinManager = _distributedSettings.UseBinManager || _distributedSettings.ProactiveCopyUsePreferredLocations,
PreferredLocationsExpiryTime = TimeSpan.FromMinutes(_distributedSettings.PreferredLocationsExpiryTimeMinutes),
PrimaryMachineLocation = OrderedResolvedCacheSettings[0].MachineLocation,
AdditionalMachineLocations = OrderedResolvedCacheSettings.Skip(1).Select(r => r.MachineLocation).ToArray()
};
ApplyIfNotNull(_distributedSettings.RedisConnectionErrorLimit, v => redisContentLocationStoreConfiguration.RedisConnectionErrorLimit = v);
ApplyIfNotNull(_distributedSettings.TraceRedisFailures, v => redisContentLocationStoreConfiguration.TraceRedisFailures = v);
ApplyIfNotNull(_distributedSettings.TraceRedisTransientFailures, v => redisContentLocationStoreConfiguration.TraceRedisTransientFailures = v);
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineExpiryMinutes, v => redisContentLocationStoreConfiguration.MachineExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.TouchFrequencyMinutes, v => redisContentLocationStoreConfiguration.TouchFrequency = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.EvictionMinAgeMinutes, v => redisContentLocationStoreConfiguration.EvictionMinAge = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.RetryWindowSeconds, v => redisContentLocationStoreConfiguration.RetryWindow = TimeSpan.FromSeconds(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(primaryCacheRoot / "LocationDb")
{
StoreClusterState = _distributedSettings.StoreClusterStateInDatabase,
LogsKeepLongTerm = true,
UseContextualEntryOperationLogging = _distributedSettings.UseContextualEntryDatabaseOperationLogging
};
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.GarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
if (_distributedSettings.EnableDistributedCache)
{
dbConfig.MetadataGarbageCollectionEnabled = true;
dbConfig.MetadataGarbageCollectionMaximumNumberOfEntriesToKeep = _distributedSettings.MaximumNumberOfMetadataEntriesToStore;
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheEnabled, v => dbConfig.ContentCacheEnabled = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushDegreeOfParallelism, v => dbConfig.FlushDegreeOfParallelism = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushTransactionSize, v => dbConfig.FlushTransactionSize = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushSingleTransaction, v => dbConfig.FlushSingleTransaction = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushPreservePercentInMemory, v => dbConfig.FlushPreservePercentInMemory = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheMaximumUpdatesPerFlush, v => dbConfig.CacheMaximumUpdatesPerFlush = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheFlushingMaximumInterval, v => dbConfig.CacheFlushingMaximumInterval = v);
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionIntervalMinutes,
v => dbConfig.FullRangeCompactionInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionVariant,
v => {
if (Enum.TryParse <FullRangeCompactionVariant>(v, out var variant))
{
dbConfig.FullRangeCompactionVariant = variant;
}
else
{
throw new ArgumentException($"Failed to parse `{nameof(_distributedSettings.FullRangeCompactionVariant)}` setting with value `{_distributedSettings.FullRangeCompactionVariant}` into type `{nameof(FullRangeCompactionVariant)}`");
}
});
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionByteIncrementStep,
v => dbConfig.FullRangeCompactionByteIncrementStep = v);
if (_distributedSettings.ContentLocationDatabaseLogsBackupEnabled)
{
dbConfig.LogsBackupPath = primaryCacheRoot / "LocationDbLogs";
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseLogsBackupRetentionMinutes, v => dbConfig.LogsRetention = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseEnumerateSortedKeysFromStorageBufferSize, v => dbConfig.EnumerateSortedKeysFromStorageBufferSize = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseEnumerateEntriesWithSortedKeysFromStorageBufferSize, v => dbConfig.EnumerateEntriesWithSortedKeysFromStorageBufferSize = v);
redisContentLocationStoreConfiguration.Database = dbConfig;
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, primaryCacheRoot, dbConfig).GetAwaiter().GetResult();
}
if (_distributedSettings.IsRedisGarbageCollectionEnabled)
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = new RedisGarbageCollectionConfiguration()
{
MaximumEntryLastAccessTime = TimeSpan.FromMinutes(30)
};
}
else
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = null;
}
_arguments.Overrides.Override(redisContentLocationStoreConfiguration);
ConfigurationPrinter.TraceConfiguration(redisContentLocationStoreConfiguration, _logger);
return(redisContentLocationStoreConfiguration);
}
public RedisMemoizationStoreFactory CreateRedisCacheFactory(AbsolutePath localCacheRoot, out RedisContentLocationStoreConfiguration config)
{
var redisContentLocationStoreConfiguration = new RedisMemoizationStoreConfiguration
{
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
EvictionWindowSize = _distributedSettings.EvictionWindowSize,
EvictionPoolSize = _distributedSettings.EvictionPoolSize,
EvictionRemovalFraction = _distributedSettings.EvictionRemovalFraction,
EvictionDiscardFraction = _distributedSettings.EvictionDiscardFraction,
UseTieredDistributedEviction = _distributedSettings.UseTieredDistributedEviction,
MemoizationExpiryTime = TimeSpan.FromMinutes(_distributedSettings.RedisMemoizationExpiryTimeMinutes),
ProactiveCopyLocationsThreshold = _distributedSettings.ProactiveCopyLocationsThreshold,
PreferredLocationsExpiryTime = TimeSpan.FromMinutes(_distributedSettings.PreferredLocationsExpiryTimeMinutes),
DelayForProactiveReplication = TimeSpan.FromSeconds(_distributedSettings.ProactiveReplicationDelaySeconds),
ProactiveReplicationCopyLimit = _distributedSettings.ProactiveReplicationCopyLimit,
EnableProactiveReplication = _distributedSettings.EnableProactiveReplication,
};
ApplyIfNotNull(_distributedSettings.RedisConnectionErrorLimit, v => redisContentLocationStoreConfiguration.RedisConnectionErrorLimit = v);
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineExpiryMinutes, v => redisContentLocationStoreConfiguration.MachineExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.TouchFrequencyMinutes, v => redisContentLocationStoreConfiguration.TouchFrequency = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.EvictionMinAgeMinutes, v => redisContentLocationStoreConfiguration.EvictionMinAge = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.RetryWindowSeconds, v => redisContentLocationStoreConfiguration.RetryWindow = TimeSpan.FromSeconds(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(localCacheRoot / "LocationDb")
{
StoreClusterState = _distributedSettings.StoreClusterStateInDatabase,
LogsKeepLongTerm = true,
};
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.GarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
if (_distributedSettings.EnableDistributedCache)
{
dbConfig.MetadataGarbageCollectionEnabled = true;
dbConfig.MetadataGarbageCollectionMaximumNumberOfEntriesToKeep = _distributedSettings.MaximumNumberOfMetadataEntriesToStore;
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheEnabled, v => dbConfig.ContentCacheEnabled = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushDegreeOfParallelism, v => dbConfig.FlushDegreeOfParallelism = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushTransactionSize, v => dbConfig.FlushTransactionSize = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushSingleTransaction, v => dbConfig.FlushSingleTransaction = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushPreservePercentInMemory, v => dbConfig.FlushPreservePercentInMemory = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheMaximumUpdatesPerFlush, v => dbConfig.CacheMaximumUpdatesPerFlush = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheFlushingMaximumInterval, v => dbConfig.CacheFlushingMaximumInterval = v);
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionIntervalMinutes,
v => dbConfig.FullRangeCompactionInterval = TimeSpan.FromMinutes(v));
if (_distributedSettings.ContentLocationDatabaseLogsBackupEnabled)
{
dbConfig.LogsBackupPath = localCacheRoot / "LocationDbLogs";
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseLogsBackupRetentionMinutes, v => dbConfig.LogsRetention = TimeSpan.FromMinutes(v));
redisContentLocationStoreConfiguration.Database = dbConfig;
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, localCacheRoot, dbConfig).GetAwaiter().GetResult();
}
if (_distributedSettings.IsRedisGarbageCollectionEnabled)
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = new RedisGarbageCollectionConfiguration()
{
MaximumEntryLastAccessTime = TimeSpan.FromMinutes(30)
};
}
else
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = null;
}
var contentHashBumpTime = TimeSpan.FromMinutes(_distributedSettings.ContentHashBumpTimeMinutes);
// RedisContentSecretName and RedisMachineLocationsSecretName can be null. HostConnectionStringProvider won't fail in this case.
IConnectionStringProvider contentConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisContentSecretName);
IConnectionStringProvider machineLocationsConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisMachineLocationsSecretName);
_arguments.Overrides.Override(redisContentLocationStoreConfiguration);
config = redisContentLocationStoreConfiguration;
return(new RedisMemoizationStoreFactory(
contentConnectionStringProvider,
machineLocationsConnectionStringProvider,
_arguments.Overrides.Clock,
contentHashBumpTime,
_keySpace,
configuration: redisContentLocationStoreConfiguration
));
}
