/// <nodoc />
internal RedisGlobalStore(
IClock clock,
RedisContentLocationStoreConfiguration configuration,
RedisDatabaseAdapter primaryRedisDb,
RedisDatabaseAdapter secondaryRedisDb,
RedisDatabaseAdapter primaryRedisBlobDb,
RedisDatabaseAdapter secondaryRedisBlobDb,
IMasterElectionMechanism masterElectionMechanism)
{
Contract.Requires(configuration.CentralStore != null);
_clock = clock;
Configuration = configuration;
RaidedRedis = new RaidedRedisDatabase(Tracer, primaryRedisDb, secondaryRedisDb);
var checkpointKeyBase = configuration.CentralStore.CentralStateKeyBase;
_clusterStateKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".ClusterState", this, _clock, RaidedRedis);
MemoizationAdapter = new RedisMemoizationAdapter(RaidedRedis, configuration.Memoization);
PrimaryBlobAdapter = new RedisBlobAdapter(primaryRedisBlobDb, _clock, Configuration);
SecondaryBlobAdapter = new RedisBlobAdapter(secondaryRedisBlobDb, _clock, Configuration);
_masterElectionMechanism = masterElectionMechanism;
}
private static DistributedContentStoreSettings CreateDistributedStoreSettings(
DistributedCacheServiceArguments arguments,
RedisContentLocationStoreConfiguration redisContentLocationStoreConfiguration)
{
var distributedSettings = arguments.Configuration.DistributedContentSettings;
PinConfiguration pinConfiguration = new PinConfiguration();
if (distributedSettings.IsPinBetterEnabled)
{
ApplyIfNotNull(distributedSettings.PinMinUnverifiedCount, v => pinConfiguration.PinMinUnverifiedCount = v);
ApplyIfNotNull(distributedSettings.StartCopyWhenPinMinUnverifiedCountThreshold, v => pinConfiguration.StartCopyWhenPinMinUnverifiedCountThreshold = v);
ApplyIfNotNull(distributedSettings.MaxIOOperations, v => pinConfiguration.MaxIOOperations = v);
}
var distributedContentStoreSettings = new DistributedContentStoreSettings()
{
TrustedHashFileSizeBoundary = distributedSettings.TrustedHashFileSizeBoundary,
ParallelHashingFileSizeBoundary = distributedSettings.ParallelHashingFileSizeBoundary,
MaxConcurrentCopyOperations = distributedSettings.MaxConcurrentCopyOperations,
PinConfiguration = pinConfiguration,
RetryIntervalForCopies = distributedSettings.RetryIntervalForCopies,
MaxRetryCount = distributedSettings.MaxRetryCount,
TimeoutForProactiveCopies = TimeSpan.FromMinutes(distributedSettings.TimeoutForProactiveCopiesMinutes),
ProactiveCopyMode = (ProactiveCopyMode)Enum.Parse(typeof(ProactiveCopyMode), distributedSettings.ProactiveCopyMode),
PushProactiveCopies = distributedSettings.PushProactiveCopies,
ProactiveCopyOnPut = distributedSettings.ProactiveCopyOnPut,
ProactiveCopyOnPin = distributedSettings.ProactiveCopyOnPin,
ProactiveCopyUsePreferredLocations = distributedSettings.ProactiveCopyUsePreferredLocations,
MaxConcurrentProactiveCopyOperations = distributedSettings.MaxConcurrentProactiveCopyOperations,
ProactiveCopyLocationsThreshold = distributedSettings.ProactiveCopyLocationsThreshold,
ProactiveCopyRejectOldContent = distributedSettings.ProactiveCopyRejectOldContent,
ReplicaCreditInMinutes = distributedSettings.IsDistributedEvictionEnabled ? distributedSettings.ReplicaCreditInMinutes : null,
EnableRepairHandling = distributedSettings.IsRepairHandlingEnabled,
LocationStoreBatchSize = distributedSettings.RedisBatchPageSize,
RestrictedCopyReplicaCount = distributedSettings.RestrictedCopyReplicaCount,
CopyAttemptsWithRestrictedReplicas = distributedSettings.CopyAttemptsWithRestrictedReplicas,
AreBlobsSupported = redisContentLocationStoreConfiguration.AreBlobsSupported,
MaxBlobSize = redisContentLocationStoreConfiguration.MaxBlobSize,
DelayForProactiveReplication = TimeSpan.FromSeconds(distributedSettings.ProactiveReplicationDelaySeconds),
ProactiveReplicationCopyLimit = distributedSettings.ProactiveReplicationCopyLimit,
EnableProactiveReplication = distributedSettings.EnableProactiveReplication,
TraceProactiveCopy = distributedSettings.TraceProactiveCopy,
ProactiveCopyGetBulkBatchSize = distributedSettings.ProactiveCopyGetBulkBatchSize,
ProactiveCopyGetBulkInterval = TimeSpan.FromSeconds(distributedSettings.ProactiveCopyGetBulkIntervalSeconds)
};
if (distributedSettings.EnableProactiveReplication && redisContentLocationStoreConfiguration.Checkpoint != null)
{
distributedContentStoreSettings.ProactiveReplicationInterval = redisContentLocationStoreConfiguration.Checkpoint.RestoreCheckpointInterval;
}
ApplyIfNotNull(distributedSettings.MaximumConcurrentPutAndPlaceFileOperations, v => distributedContentStoreSettings.MaximumConcurrentPutAndPlaceFileOperations = v);
arguments.Overrides.Override(distributedContentStoreSettings);
ConfigurationPrinter.TraceConfiguration(distributedContentStoreSettings, arguments.Logger);
return(distributedContentStoreSettings);
}
public TransitioningGlobalCacheStore(RedisContentLocationStoreConfiguration configuration, IGlobalCacheStore redisStore, IGlobalCacheStore distributedStore)
{
Contract.Requires(configuration.AllContentMetadataStoreModeFlags.HasAnyFlag(ContentMetadataStoreModeFlags.Redis) &&
configuration.AllContentMetadataStoreModeFlags.HasAnyFlag(ContentMetadataStoreModeFlags.Distributed),
"Transitioning store should not used for cases where one store or the other is used exclusively");
_configuration = configuration;
_redisStore = redisStore;
_distributedStore = distributedStore;
if (BlobMode.HasAllFlags(ContentMetadataStoreModeFlags.PreferRedis) || BlobMode.MaskFlags(ContentMetadataStoreModeFlags.Distributed) == 0)
{
AreBlobsSupported = _redisStore.AreBlobsSupported;
}
else if (BlobMode.HasAllFlags(ContentMetadataStoreModeFlags.PreferDistributed) || BlobMode.MaskFlags(ContentMetadataStoreModeFlags.Redis) == 0)
{
AreBlobsSupported = _distributedStore.AreBlobsSupported;
}
else
{
AreBlobsSupported = _redisStore.AreBlobsSupported || _distributedStore.AreBlobsSupported;
}
// Mask used to only include valid flags for BlobMode based on blob support in the respective stores
_blobSupportedMask = ContentMetadataStoreModeFlags.All
.Subtract(_redisStore.AreBlobsSupported ? 0 : ContentMetadataStoreModeFlags.Redis | ContentMetadataStoreModeFlags.PreferRedis)
.Subtract(_distributedStore.AreBlobsSupported ? 0 : ContentMetadataStoreModeFlags.Distributed | ContentMetadataStoreModeFlags.PreferDistributed);
}
public MockContentLocationStoreFactory(
ITestRedisDatabase primaryRedisDatabase,
ITestRedisDatabase secondaryRedisDatabase,
AbsolutePath rootDirectory,
ITestClock mockClock = null,
RedisContentLocationStoreConfiguration configuration = null)
: base(mockClock ?? TestSystemClock.Instance, configuration ?? CreateDefaultConfiguration(rootDirectory), CreateTestCopier(mockClock ?? TestSystemClock.Instance, rootDirectory))
{
_primaryRedisDatabase = primaryRedisDatabase;
_secondaryRedisDatabase = secondaryRedisDatabase;
}
public TransitioningContentMetadataStore(RedisContentLocationStoreConfiguration configuration, IContentMetadataStore redisStore, IContentMetadataStore distributedStore)
{
Contract.Requires(configuration.ContentMetadataStoreMode != ContentMetadataStoreMode.Redis &&
configuration.ContentMetadataStoreMode != ContentMetadataStoreMode.Distributed,
"Transitioning store should not used for cases where one store or the other is used exclusively");
_configuration = configuration;
_redisStore = redisStore;
_distributedStore = distributedStore;
_preferDistributed = configuration.ContentMetadataStoreModeFlags.HasFlag(ContentMetadataStoreModeFlags.PreferDistributed);
}
public MockRedisContentLocationStoreFactory(
ITestRedisDatabase redisDatabase,
ITestRedisDatabase machineLocationRedisDatabase,
AbsolutePath localCacheRoot,
IClock mockClock = null,
RedisContentLocationStoreConfiguration configuration = null)
{
_mockClock = mockClock ?? SystemClock.Instance;
RedisDatabase = redisDatabase;
MachineLocationRedisDatabase = machineLocationRedisDatabase;
_localMachineLocation = new MachineLocation(localCacheRoot.Path);
_configuration = configuration ?? RedisContentLocationStoreConfiguration.Default;
}
/// <nodoc />
public TransitioningContentLocationStore(
RedisContentLocationStoreConfiguration configuration,
RedisContentLocationStore redisContentLocationStore,
LocalLocationStore localLocationStore)
{
Contract.Requires(configuration != null);
_configuration = configuration;
_localLocationStore = localLocationStore;
_redisContentLocationStore = redisContentLocationStore;
Contract.Assert(!_configuration.HasReadOrWriteMode(ContentLocationMode.Redis) || _redisContentLocationStore != null);
Contract.Assert(!_configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore) || _localLocationStore != null);
}
public RocksDbRedisContentLocationStoreTests(ITestOutputHelper output, LocalRedisFixture redis)
: base(redis, output)
{
// Need to use unique folder for each test instance, because more then one test may be executed simultaneously.
var uniqueOutputFolder = Guid.NewGuid().ToString();
_workingDirectory = new DisposableDirectory(new PassThroughFileSystem(TestGlobal.Logger), Path.Combine(uniqueOutputFolder, "redis"));
DefaultConfiguration = new RedisContentLocationStoreConfiguration()
{
Database = new RocksDbContentLocationDatabaseConfiguration(_workingDirectory.Path / "rocksdb")
};
output.WriteLine($"Tests output folder is '{_workingDirectory.Path}'");
}
/// <nodoc />
public TransitioningContentLocationStore(
RedisContentLocationStoreConfiguration configuration,
LocalLocationStore localLocationStore,
MachineLocation localMachineLocation,
ILocalContentStore localContentStore)
{
Contract.Requires(configuration != null);
Contract.Requires(localMachineLocation.IsValid);
LocalContentStore = localContentStore;
_configuration = configuration;
LocalLocationStore = localLocationStore;
LocalMachineLocation = localMachineLocation;
}
public void MaskConfigurationProperties()
{
var testConfig = new RedisContentLocationStoreConfiguration();
var firstConnectionString = "testGlobalString";
var secondConnectionString = "password12345";
var blobConnectionString = "blobConnectionString";
testConfig.RedisGlobalStoreConnectionString = firstConnectionString;
testConfig.RedisGlobalStoreSecondaryConnectionString = secondConnectionString;
testConfig.CentralStore = new BlobCentralStoreConfiguration(new AzureBlobStorageCredentials(blobConnectionString), "testContainer", "testKey");
string configString = ConfigurationPrinter.ConfigToString(testConfig);
configString.Should().NotContain(firstConnectionString);
configString.Should().NotContain(secondConnectionString);
configString.Should().NotContain(blobConnectionString);
}
/// <nodoc />
public RedisGlobalStore(IClock clock, RedisContentLocationStoreConfiguration configuration, MachineLocation localMachineLocation, RedisDatabaseAdapter primaryRedisDb, RedisDatabaseAdapter secondaryRedisDb)
{
Contract.Requires(configuration.CentralStore != null);
_clock = clock;
_configuration = configuration;
_raidedRedis = new RaidedRedisDatabase(Tracer, primaryRedisDb, secondaryRedisDb);
var checkpointKeyBase = configuration.CentralStore.CentralStateKeyBase;
_checkpointsKey = new ReplicatedRedisHashKey(configuration.GetCheckpointPrefix() + ".Checkpoints", this, _clock, _raidedRedis);
_masterLeaseKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".MasterLease", this, _clock, _raidedRedis);
_clusterStateKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".ClusterState", this, _clock, _raidedRedis);
LocalMachineLocation = localMachineLocation;
_blobAdapter = new RedisBlobAdapter(_raidedRedis.PrimaryRedisDb, TimeSpan.FromMinutes(_configuration.BlobExpiryTimeMinutes), _configuration.MaxBlobCapacity, _clock, Tracer);
}
private static RedisContentLocationStoreConfiguration CreateDefaultConfiguration(AbsolutePath rootDirectory)
{
var configuration = new RedisContentLocationStoreConfiguration()
{
Keyspace = "Default:",
RedisGlobalStoreConnectionString = "Test",
};
configuration.InlinePostInitialization = true;
configuration.MachineStateRecomputeInterval = TimeSpan.Zero;
configuration.EventStore = new MemoryContentLocationEventStoreConfiguration();
configuration.Database = new RocksDbContentLocationDatabaseConfiguration(rootDirectory / "rocksdb");
configuration.Checkpoint = new CheckpointConfiguration(rootDirectory);
configuration.CentralStore = new LocalDiskCentralStoreConfiguration(rootDirectory / "centralStore", "checkpoints-key");
configuration.PrimaryMachineLocation = new MachineLocation(rootDirectory.ToString());
return(configuration);
}
/// <nodoc />
internal RedisGlobalStore(
IClock clock,
RedisContentLocationStoreConfiguration configuration,
RedisDatabaseAdapter primaryRedisDb,
RedisDatabaseAdapter secondaryRedisDb,
RedisDatabaseAdapter primaryRedisBlobDb,
RedisDatabaseAdapter secondaryRedisBlobDb)
{
Contract.Requires(configuration.CentralStore != null);
_clock = clock;
Configuration = configuration;
RaidedRedis = new RaidedRedisDatabase(Tracer, primaryRedisDb, secondaryRedisDb);
var checkpointKeyBase = configuration.CentralStore.CentralStateKeyBase;
_checkpointsKey = new ReplicatedRedisHashKey(configuration.GetCheckpointPrefix() + ".Checkpoints", this, _clock, RaidedRedis);
_masterLeaseKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".MasterLease", this, _clock, RaidedRedis);
_clusterStateKey = new ReplicatedRedisHashKey(checkpointKeyBase + ".ClusterState", this, _clock, RaidedRedis);
PrimaryBlobAdapter = new RedisBlobAdapter(primaryRedisBlobDb, _clock, Configuration);
SecondaryBlobAdapter = new RedisBlobAdapter(secondaryRedisBlobDb, _clock, Configuration);
}
public override void Override(RedisContentLocationStoreConfiguration configuration)
{
configuration.InlinePostInitialization = true;
// Set recompute time to zero to force recomputation on every heartbeat
configuration.MachineStateRecomputeInterval = TimeSpan.Zero;
if (!_tests.UseRealEventHub)
{
// Propagate epoch from normal configuration to in-memory configuration
_tests.MemoryEventStoreConfiguration.Epoch = configuration.EventStore.Epoch;
configuration.EventStore = _tests.MemoryEventStoreConfiguration;
}
if (configuration.CentralStore is BlobCentralStoreConfiguration blobConfig)
{
blobConfig.EnableGarbageCollect = false;
}
_tests._overrideRedis?.Invoke(configuration);
_tests._configurations[_storeIndex] = configuration;
}
private async Task ApplySecretSettingsForLlsAsync(
RedisContentLocationStoreConfiguration configuration,
AbsolutePath localCacheRoot,
RocksDbContentLocationDatabaseConfiguration dbConfig)
{
(var secrets, var errors) = await _secretRetriever.TryRetrieveSecretsAsync();
if (secrets == null)
{
_logger.Error($"Unable to configure Local Location Store. {errors}");
return;
}
configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot);
if (_distributedSettings.IsMasterEligible)
{
// Use master selection by setting role to null
configuration.Checkpoint.Role = null;
}
else
{
// Not master eligible. Set role to worker.
configuration.Checkpoint.Role = Role.Worker;
}
// It is important to set the current role of the service, to have non-null Role column
// in all the tracing messages emitted to Kusto.
GlobalInfoStorage.SetServiceRole(configuration.Checkpoint.Role?.ToString() ?? "MasterEligible");
var checkpointConfiguration = configuration.Checkpoint;
ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value);
ApplyIfNotNull(
_distributedSettings.HeartbeatIntervalMinutes,
value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.CreateCheckpointIntervalMinutes,
value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.RestoreCheckpointIntervalMinutes,
value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.RestoreCheckpointTimeoutMinutes,
value => checkpointConfiguration.RestoreCheckpointTimeout = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.UpdateClusterStateIntervalSeconds,
value => checkpointConfiguration.UpdateClusterStateInterval = TimeSpan.FromSeconds(value));
ApplyIfNotNull(_distributedSettings.PacemakerEnabled, v => checkpointConfiguration.PacemakerEnabled = v);
ApplyIfNotNull(_distributedSettings.PacemakerNumberOfBuckets, v => checkpointConfiguration.PacemakerNumberOfBuckets = v);
ApplyIfNotNull(_distributedSettings.PacemakerUseRandomIdentifier, v => checkpointConfiguration.PacemakerUseRandomIdentifier = v);
ApplyIfNotNull(
_distributedSettings.SafeToLazilyUpdateMachineCountThreshold,
value => configuration.SafeToLazilyUpdateMachineCountThreshold = value);
configuration.EnableReconciliation = !_distributedSettings.Unsafe_DisableReconciliation;
configuration.ReconciliationCycleFrequency = TimeSpan.FromMinutes(_distributedSettings.ReconciliationCycleFrequencyMinutes);
configuration.ReconciliationMaxCycleSize = _distributedSettings.ReconciliationMaxCycleSize;
configuration.ReconciliationMaxRemoveHashesCycleSize = _distributedSettings.ReconciliationMaxRemoveHashesCycleSize;
configuration.ReconciliationMaxRemoveHashesAddPercentage = _distributedSettings.ReconciliationMaxRemoveHashesAddPercentage;
ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value);
ApplyIfNotNull(_distributedSettings.IncrementalCheckpointDegreeOfParallelism, value => configuration.Checkpoint.IncrementalCheckpointDegreeOfParallelism = value);
ApplyIfNotNull(_distributedSettings.UseRedisPreventThreadTheftFeature, value => configuration.UsePreventThreadTheftFeature = value);
configuration.RedisGlobalStoreConnectionString = ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName)).Secret;
if (_distributedSettings.SecondaryGlobalRedisSecretName != null)
{
configuration.RedisGlobalStoreSecondaryConnectionString = ((PlainTextSecret)GetRequiredSecret(
secrets,
_distributedSettings.SecondaryGlobalRedisSecretName)).Secret;
}
ApplyIfNotNull(_distributedSettings.RedisInternalLogSeverity, value =>
{
if (!Enum.TryParse <Severity>(value, out var parsedValue))
{
throw new ArgumentException($"Failed to parse `{nameof(_distributedSettings.RedisInternalLogSeverity)}` setting with value `{value}` into type `{nameof(Severity)}`");
}
configuration.RedisInternalLogSeverity = parsedValue;
});
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value));
ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => configuration.Checkpoint.RestoreCheckpointAgeThreshold = TimeSpan.FromMinutes(v));
// Need to disable cleaning database on initialization when restore checkpoint age is set.
ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => dbConfig.CleanOnInitialize = false);
var errorBuilder = new StringBuilder();
var storageCredentials = GetStorageCredentials(secrets, errorBuilder);
Contract.Assert(storageCredentials != null && storageCredentials.Length > 0);
var blobStoreConfiguration = new BlobCentralStoreConfiguration(
credentials: storageCredentials,
containerName: _arguments.HostInfo.AppendRingSpecifierIfNeeded("checkpoints", _distributedSettings.UseRingIsolation),
checkpointsKey: "checkpoints-eventhub");
ApplyIfNotNull(
_distributedSettings.CentralStorageOperationTimeoutInMinutes,
value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value));
configuration.CentralStore = blobStoreConfiguration;
if (_distributedSettings.UseDistributedCentralStorage)
{
var distributedCentralStoreConfiguration = new DistributedCentralStoreConfiguration(localCacheRoot)
{
MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb,
PropagationDelay = TimeSpan.FromSeconds(_distributedSettings.CentralStoragePropagationDelaySeconds),
PropagationIterations = _distributedSettings.CentralStoragePropagationIterations,
MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies
};
if (_distributedSettings.UseSelfCheckSettingsForDistributedCentralStorage)
{
distributedCentralStoreConfiguration.SelfCheckSettings = CreateSelfCheckSettings(_distributedSettings);
}
distributedCentralStoreConfiguration.TraceFileSystemContentStoreDiagnosticMessages = _distributedSettings.TraceFileSystemContentStoreDiagnosticMessages;
ApplyIfNotNull(_distributedSettings.DistributedCentralStoragePeerToPeerCopyTimeoutSeconds, v => distributedCentralStoreConfiguration.PeerToPeerCopyTimeout = TimeSpan.FromSeconds(v));
ApplyIfNotNull(_distributedSettings.DistributedCentralStorageImmutabilityOptimizations, v => distributedCentralStoreConfiguration.ImmutabilityOptimizations = v);
configuration.DistributedCentralStore = distributedCentralStoreConfiguration;
}
var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration(
eventHubName: _distributedSettings.EventHubName,
eventHubConnectionString: ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName)).Secret,
consumerGroupName: _distributedSettings.EventHubConsumerGroupName,
epoch: _keySpace + _distributedSettings.EventHubEpoch);
dbConfig.Epoch = eventStoreConfiguration.Epoch;
configuration.EventStore = eventStoreConfiguration;
ApplyIfNotNull(
_distributedSettings.MaxEventProcessingConcurrency,
value => eventStoreConfiguration.MaxEventProcessingConcurrency = value);
ApplyIfNotNull(
_distributedSettings.EventBatchSize,
value => eventStoreConfiguration.EventBatchSize = value);
ApplyIfNotNull(
_distributedSettings.EventProcessingMaxQueueSize,
value => eventStoreConfiguration.EventProcessingMaxQueueSize = value);
}
public RedisMemoizationStoreFactory CreateRedisCacheFactory(AbsolutePath localCacheRoot, out RedisContentLocationStoreConfiguration config)
{
var redisContentLocationStoreConfiguration = new RedisMemoizationStoreConfiguration
{
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
EvictionWindowSize = _distributedSettings.EvictionWindowSize,
EvictionPoolSize = _distributedSettings.EvictionPoolSize,
EvictionRemovalFraction = _distributedSettings.EvictionRemovalFraction,
EvictionDiscardFraction = _distributedSettings.EvictionDiscardFraction,
UseTieredDistributedEviction = _distributedSettings.UseTieredDistributedEviction,
MemoizationExpiryTime = TimeSpan.FromMinutes(_distributedSettings.RedisMemoizationExpiryTimeMinutes),
ProactiveCopyLocationsThreshold = _distributedSettings.ProactiveCopyLocationsThreshold,
PreferredLocationsExpiryTime = TimeSpan.FromMinutes(_distributedSettings.PreferredLocationsExpiryTimeMinutes),
DelayForProactiveReplication = TimeSpan.FromSeconds(_distributedSettings.ProactiveReplicationDelaySeconds),
ProactiveReplicationCopyLimit = _distributedSettings.ProactiveReplicationCopyLimit,
EnableProactiveReplication = _distributedSettings.EnableProactiveReplication,
};
ApplyIfNotNull(_distributedSettings.RedisConnectionErrorLimit, v => redisContentLocationStoreConfiguration.RedisConnectionErrorLimit = v);
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineExpiryMinutes, v => redisContentLocationStoreConfiguration.MachineExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.TouchFrequencyMinutes, v => redisContentLocationStoreConfiguration.TouchFrequency = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.EvictionMinAgeMinutes, v => redisContentLocationStoreConfiguration.EvictionMinAge = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.RetryWindowSeconds, v => redisContentLocationStoreConfiguration.RetryWindow = TimeSpan.FromSeconds(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(localCacheRoot / "LocationDb")
{
StoreClusterState = _distributedSettings.StoreClusterStateInDatabase,
LogsKeepLongTerm = true,
};
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.GarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
if (_distributedSettings.EnableDistributedCache)
{
dbConfig.MetadataGarbageCollectionEnabled = true;
dbConfig.MetadataGarbageCollectionMaximumNumberOfEntriesToKeep = _distributedSettings.MaximumNumberOfMetadataEntriesToStore;
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheEnabled, v => dbConfig.ContentCacheEnabled = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushDegreeOfParallelism, v => dbConfig.FlushDegreeOfParallelism = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushTransactionSize, v => dbConfig.FlushTransactionSize = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushSingleTransaction, v => dbConfig.FlushSingleTransaction = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushPreservePercentInMemory, v => dbConfig.FlushPreservePercentInMemory = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheMaximumUpdatesPerFlush, v => dbConfig.CacheMaximumUpdatesPerFlush = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheFlushingMaximumInterval, v => dbConfig.CacheFlushingMaximumInterval = v);
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionIntervalMinutes,
v => dbConfig.FullRangeCompactionInterval = TimeSpan.FromMinutes(v));
if (_distributedSettings.ContentLocationDatabaseLogsBackupEnabled)
{
dbConfig.LogsBackupPath = localCacheRoot / "LocationDbLogs";
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseLogsBackupRetentionMinutes, v => dbConfig.LogsRetention = TimeSpan.FromMinutes(v));
redisContentLocationStoreConfiguration.Database = dbConfig;
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, localCacheRoot, dbConfig).GetAwaiter().GetResult();
}
if (_distributedSettings.IsRedisGarbageCollectionEnabled)
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = new RedisGarbageCollectionConfiguration()
{
MaximumEntryLastAccessTime = TimeSpan.FromMinutes(30)
};
}
else
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = null;
}
var contentHashBumpTime = TimeSpan.FromMinutes(_distributedSettings.ContentHashBumpTimeMinutes);
// RedisContentSecretName and RedisMachineLocationsSecretName can be null. HostConnectionStringProvider won't fail in this case.
IConnectionStringProvider contentConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisContentSecretName);
IConnectionStringProvider machineLocationsConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisMachineLocationsSecretName);
_arguments.Overrides.Override(redisContentLocationStoreConfiguration);
config = redisContentLocationStoreConfiguration;
return(new RedisMemoizationStoreFactory(
contentConnectionStringProvider,
machineLocationsConnectionStringProvider,
_arguments.Overrides.Clock,
contentHashBumpTime,
_keySpace,
configuration: redisContentLocationStoreConfiguration
));
}
public IContentStore CreateContentStore(
AbsolutePath localCacheRoot,
NagleQueue <ContentHash> evictionAnnouncer = null,
ProactiveReplicationArgs replicationSettings = null,
DistributedEvictionSettings distributedEvictionSettings = null,
bool checkLocalFiles = true,
TrimBulkAsync trimBulkAsync = null)
{
var redisContentLocationStoreConfiguration = new RedisContentLocationStoreConfiguration
{
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
EvictionWindowSize = _distributedSettings.EvictionWindowSize
};
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineExpiryMinutes, v => redisContentLocationStoreConfiguration.MachineExpiry = TimeSpan.FromMinutes(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(localCacheRoot / "LocationDb")
{
StoreClusterState = _distributedSettings.StoreClusterStateInDatabase
};
redisContentLocationStoreConfiguration.Database = dbConfig;
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.LocalDatabaseGarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheEnabled, v => dbConfig.CacheEnabled = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushDegreeOfParallelism, v => dbConfig.FlushDegreeOfParallelism = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushSingleTransaction, v => dbConfig.FlushSingleTransaction = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushPreservePercentInMemory, v => dbConfig.FlushPreservePercentInMemory = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheMaximumUpdatesPerFlush, v => dbConfig.CacheMaximumUpdatesPerFlush = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheFlushingMaximumInterval, v => dbConfig.CacheFlushingMaximumInterval = v);
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, localCacheRoot).GetAwaiter().GetResult();
}
if (_distributedSettings.IsRedisGarbageCollectionEnabled)
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = new RedisGarbageCollectionConfiguration()
{
MaximumEntryLastAccessTime = TimeSpan.FromMinutes(30)
};
}
else
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = null;
}
var localMachineLocation = _arguments.PathTransformer.GetLocalMachineLocation(localCacheRoot);
var contentHashBumpTime = TimeSpan.FromMinutes(_distributedSettings.ContentHashBumpTimeMinutes);
// RedisContentSecretName and RedisMachineLocationsSecretName can be null. HostConnectionStringProvider won't fail in this case.
IConnectionStringProvider contentConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisContentSecretName);
IConnectionStringProvider machineLocationsConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisMachineLocationsSecretName);
var redisContentLocationStoreFactory = new RedisContentLocationStoreFactory(
contentConnectionStringProvider,
machineLocationsConnectionStringProvider,
SystemClock.Instance,
contentHashBumpTime,
_keySpace,
localMachineLocation,
configuration: redisContentLocationStoreConfiguration
);
ReadOnlyDistributedContentSession <AbsolutePath> .ContentAvailabilityGuarantee contentAvailabilityGuarantee;
if (string.IsNullOrEmpty(_distributedSettings.ContentAvailabilityGuarantee))
{
contentAvailabilityGuarantee =
ReadOnlyDistributedContentSession <AbsolutePath> .ContentAvailabilityGuarantee
.FileRecordsExist;
}
else if (!Enum.TryParse(_distributedSettings.ContentAvailabilityGuarantee, true, out contentAvailabilityGuarantee))
{
throw new ArgumentException($"Unable to parse {nameof(_distributedSettings.ContentAvailabilityGuarantee)}: [{_distributedSettings.ContentAvailabilityGuarantee}]");
}
PinConfiguration pinConfiguration = null;
if (_distributedSettings.IsPinBetterEnabled)
{
pinConfiguration = new PinConfiguration();
if (_distributedSettings.PinRisk.HasValue)
{
pinConfiguration.PinRisk = _distributedSettings.PinRisk.Value;
}
if (_distributedSettings.MachineRisk.HasValue)
{
pinConfiguration.MachineRisk = _distributedSettings.MachineRisk.Value;
}
if (_distributedSettings.FileRisk.HasValue)
{
pinConfiguration.FileRisk = _distributedSettings.FileRisk.Value;
}
if (_distributedSettings.MaxIOOperations.HasValue)
{
pinConfiguration.MaxIOOperations = _distributedSettings.MaxIOOperations.Value;
}
pinConfiguration.UsePinCache = _distributedSettings.IsPinCachingEnabled;
if (_distributedSettings.PinCacheReplicaCreditRetentionMinutes.HasValue)
{
pinConfiguration.PinCacheReplicaCreditRetentionMinutes = _distributedSettings.PinCacheReplicaCreditRetentionMinutes.Value;
}
if (_distributedSettings.PinCacheReplicaCreditRetentionDecay.HasValue)
{
pinConfiguration.PinCacheReplicaCreditRetentionDecay = _distributedSettings.PinCacheReplicaCreditRetentionDecay.Value;
}
}
var lazyTouchContentHashBumpTime = _distributedSettings.IsTouchEnabled ? (TimeSpan?)contentHashBumpTime : null;
if (redisContentLocationStoreConfiguration.ReadMode == ContentLocationMode.LocalLocationStore)
{
// LocalLocationStore has its own internal notion of lazy touch/registration. We disable the lazy touch in distributed content store
// because it can conflict with behavior of the local location store.
lazyTouchContentHashBumpTime = null;
}
var contentStoreSettings = FromDistributedSettings(_distributedSettings);
ConfigurationModel configurationModel = null;
if (_arguments.Configuration.LocalCasSettings.CacheSettingsByCacheName.TryGetValue(_arguments.Configuration.LocalCasSettings.CasClientSettings.DefaultCacheName, out var namedCacheSettings))
{
configurationModel = new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota(namedCacheSettings.CacheSizeQuotaString)));
}
_logger.Debug("Creating a distributed content store for Autopilot");
var contentStore =
new DistributedContentStore <AbsolutePath>(
localMachineLocation,
(announcer, evictionSettings, checkLocal, trimBulk) =>
ContentStoreFactory.CreateContentStore(_fileSystem, localCacheRoot, announcer, distributedEvictionSettings: evictionSettings,
contentStoreSettings: contentStoreSettings, trimBulkAsync: trimBulk, configurationModel: configurationModel),
redisContentLocationStoreFactory,
_arguments.Copier,
_arguments.Copier,
_arguments.PathTransformer,
contentAvailabilityGuarantee,
localCacheRoot,
_fileSystem,
_distributedSettings.RedisBatchPageSize,
new DistributedContentStoreSettings()
{
UseTrustedHash = _distributedSettings.UseTrustedHash,
CleanRandomFilesAtRoot = _distributedSettings.CleanRandomFilesAtRoot,
TrustedHashFileSizeBoundary = _distributedSettings.TrustedHashFileSizeBoundary,
ParallelHashingFileSizeBoundary = _distributedSettings.ParallelHashingFileSizeBoundary,
MaxConcurrentCopyOperations = _distributedSettings.MaxConcurrentCopyOperations,
PinConfiguration = pinConfiguration,
EmptyFileHashShortcutEnabled = _distributedSettings.EmptyFileHashShortcutEnabled,
RetryIntervalForCopies = _distributedSettings.RetryIntervalForCopies,
},
replicaCreditInMinutes: _distributedSettings.IsDistributedEvictionEnabled?_distributedSettings.ReplicaCreditInMinutes: null,
enableRepairHandling: _distributedSettings.IsRepairHandlingEnabled,
contentHashBumpTime: lazyTouchContentHashBumpTime,
contentStoreSettings: contentStoreSettings);
_logger.Debug("Created Distributed content store.");
return(contentStore);
}
public virtual void Override(RedisContentLocationStoreConfiguration configuration)
{
}
private static DistributedContentStoreSettings CreateDistributedStoreSettings(
DistributedCacheServiceArguments arguments,
RedisContentLocationStoreConfiguration redisContentLocationStoreConfiguration)
{
var distributedSettings = arguments.Configuration.DistributedContentSettings;
ContentAvailabilityGuarantee contentAvailabilityGuarantee;
if (string.IsNullOrEmpty(distributedSettings.ContentAvailabilityGuarantee))
{
contentAvailabilityGuarantee = ContentAvailabilityGuarantee.FileRecordsExist;
}
else if (!Enum.TryParse(distributedSettings.ContentAvailabilityGuarantee, true, out contentAvailabilityGuarantee))
{
throw new ArgumentException($"Unable to parse {nameof(distributedSettings.ContentAvailabilityGuarantee)}: [{distributedSettings.ContentAvailabilityGuarantee}]");
}
PinConfiguration pinConfiguration = null;
if (distributedSettings.IsPinBetterEnabled)
{
pinConfiguration = new PinConfiguration();
ApplyIfNotNull(distributedSettings.PinRisk, v => pinConfiguration.PinRisk = v);
ApplyIfNotNull(distributedSettings.PinMinUnverifiedCount, v => pinConfiguration.PinMinUnverifiedCount = v);
ApplyIfNotNull(distributedSettings.MachineRisk, v => pinConfiguration.MachineRisk = v);
ApplyIfNotNull(distributedSettings.FileRisk, v => pinConfiguration.FileRisk = v);
ApplyIfNotNull(distributedSettings.MaxIOOperations, v => pinConfiguration.MaxIOOperations = v);
pinConfiguration.IsPinCachingEnabled = distributedSettings.IsPinCachingEnabled;
ApplyIfNotNull(distributedSettings.PinCacheReplicaCreditRetentionMinutes, v => pinConfiguration.PinCachePerReplicaRetentionCreditMinutes = v);
ApplyIfNotNull(distributedSettings.PinCacheReplicaCreditRetentionDecay, v => pinConfiguration.PinCacheReplicaCreditRetentionFactor = v);
}
var contentHashBumpTime = TimeSpan.FromMinutes(distributedSettings.ContentHashBumpTimeMinutes);
var lazyTouchContentHashBumpTime = distributedSettings.IsTouchEnabled ? (TimeSpan?)contentHashBumpTime : null;
if (redisContentLocationStoreConfiguration.ReadMode == ContentLocationMode.LocalLocationStore)
{
// LocalLocationStore has its own internal notion of lazy touch/registration. We disable the lazy touch in distributed content store
// because it can conflict with behavior of the local location store.
lazyTouchContentHashBumpTime = null;
}
var distributedContentStoreSettings = new DistributedContentStoreSettings()
{
TrustedHashFileSizeBoundary = distributedSettings.TrustedHashFileSizeBoundary,
ParallelHashingFileSizeBoundary = distributedSettings.ParallelHashingFileSizeBoundary,
MaxConcurrentCopyOperations = distributedSettings.MaxConcurrentCopyOperations,
PinConfiguration = pinConfiguration,
RetryIntervalForCopies = distributedSettings.RetryIntervalForCopies,
MaxRetryCount = distributedSettings.MaxRetryCount,
TimeoutForProactiveCopies = TimeSpan.FromMinutes(distributedSettings.TimeoutForProactiveCopiesMinutes),
ProactiveCopyMode = (ProactiveCopyMode)Enum.Parse(typeof(ProactiveCopyMode), distributedSettings.ProactiveCopyMode),
PushProactiveCopies = distributedSettings.PushProactiveCopies,
ProactiveCopyOnPut = distributedSettings.ProactiveCopyOnPut,
ProactiveCopyOnPin = distributedSettings.ProactiveCopyOnPin,
ProactiveCopyUsePreferredLocations = distributedSettings.ProactiveCopyUsePreferredLocations,
MaxConcurrentProactiveCopyOperations = distributedSettings.MaxConcurrentProactiveCopyOperations,
ProactiveCopyLocationsThreshold = distributedSettings.ProactiveCopyLocationsThreshold,
ProactiveCopyRejectOldContent = distributedSettings.ProactiveCopyRejectOldContent,
ReplicaCreditInMinutes = distributedSettings.IsDistributedEvictionEnabled ? distributedSettings.ReplicaCreditInMinutes : null,
EnableRepairHandling = distributedSettings.IsRepairHandlingEnabled,
ContentHashBumpTime = lazyTouchContentHashBumpTime,
LocationStoreBatchSize = distributedSettings.RedisBatchPageSize,
ContentAvailabilityGuarantee = contentAvailabilityGuarantee,
PrioritizeDesignatedLocationsOnCopies = distributedSettings.PrioritizeDesignatedLocationsOnCopies,
RestrictedCopyReplicaCount = distributedSettings.RestrictedCopyReplicaCount,
CopyAttemptsWithRestrictedReplicas = distributedSettings.CopyAttemptsWithRestrictedReplicas,
AreBlobsSupported = redisContentLocationStoreConfiguration.AreBlobsSupported,
MaxBlobSize = redisContentLocationStoreConfiguration.MaxBlobSize,
DelayForProactiveReplication = TimeSpan.FromSeconds(distributedSettings.ProactiveReplicationDelaySeconds),
ProactiveReplicationCopyLimit = distributedSettings.ProactiveReplicationCopyLimit,
EnableProactiveReplication = distributedSettings.EnableProactiveReplication,
TraceProactiveCopy = distributedSettings.TraceProactiveCopy,
};
if (distributedSettings.EnableProactiveReplication && redisContentLocationStoreConfiguration.Checkpoint != null)
{
distributedContentStoreSettings.ProactiveReplicationInterval = redisContentLocationStoreConfiguration.Checkpoint.RestoreCheckpointInterval;
}
ApplyIfNotNull(distributedSettings.MaximumConcurrentPutAndPlaceFileOperations, v => distributedContentStoreSettings.MaximumConcurrentPutAndPlaceFileOperations = v);
arguments.Overrides.Override(distributedContentStoreSettings);
ConfigurationPrinter.TraceConfiguration(distributedContentStoreSettings, arguments.Logger);
return(distributedContentStoreSettings);
}
private async Task ApplyKeyVaultSettingsForLlsAsync(RedisContentLocationStoreConfiguration configuration, AbsolutePath localCacheRoot)
{
var errorBuilder = new StringBuilder();
var secrets = await TryRetrieveKeyVaultSecretsAsync(CancellationToken.None, errorBuilder);
if (secrets == null)
{
_logger.Error($"Unable to configure Local Location Store. {errorBuilder}");
return;
}
configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot);
if (_distributedSettings.IsMasterEligible)
{
// Use master selection by setting role to null
configuration.Checkpoint.Role = null;
}
else
{
// Not master eligible. Set role to worker.
configuration.Checkpoint.Role = Role.Worker;
}
var checkpointConfiguration = configuration.Checkpoint;
ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value);
ApplyIfNotNull(
_distributedSettings.HeartbeatIntervalMinutes,
value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.CreateCheckpointIntervalMinutes,
value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.RestoreCheckpointIntervalMinutes,
value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.SafeToLazilyUpdateMachineCountThreshold,
value => configuration.SafeToLazilyUpdateMachineCountThreshold = value);
ApplyIfNotNull(_distributedSettings.IsReconciliationEnabled, value => configuration.EnableReconciliation = value);
ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value);
configuration.RedisGlobalStoreConnectionString = GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName);
if (_distributedSettings.SecondaryGlobalRedisSecretName != null)
{
configuration.RedisGlobalStoreSecondaryConnectionString = GetRequiredSecret(
secrets,
_distributedSettings.SecondaryGlobalRedisSecretName);
}
ApplyIfNotNull(
_distributedSettings.ContentLocationReadMode,
value => configuration.ReadMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value));
ApplyIfNotNull(
_distributedSettings.ContentLocationWriteMode,
value => configuration.WriteMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value));
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value));
var storageConnectionStrings = GetStorageConnectionStrings(secrets, errorBuilder);
// We already retrieved storage connection strings, so the result should not be null.
Contract.Assert(storageConnectionStrings != null);
var blobStoreConfiguration = new BlobCentralStoreConfiguration(
connectionStrings: storageConnectionStrings,
containerName: "checkpoints",
checkpointsKey: "checkpoints-eventhub");
ApplyIfNotNull(
_distributedSettings.CentralStorageOperationTimeoutInMinutes,
value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value));
configuration.CentralStore = blobStoreConfiguration;
if (_distributedSettings.UseDistributedCentralStorage)
{
configuration.DistributedCentralStore = new DistributedCentralStoreConfiguration(localCacheRoot)
{
MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb,
PropagationDelay = TimeSpan.FromSeconds(
_distributedSettings.CentralStoragePropagationDelaySeconds),
PropagationIterations = _distributedSettings.CentralStoragePropagationIterations,
MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies
};
}
var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration(
eventHubName: "eventhub",
eventHubConnectionString: GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName),
consumerGroupName: "$Default",
epoch: _keySpace + _distributedSettings.EventHubEpoch);
configuration.EventStore = eventStoreConfiguration;
ApplyIfNotNull(
_distributedSettings.MaxEventProcessingConcurrency,
value => eventStoreConfiguration.MaxEventProcessingConcurrency = value);
}
private async Task ApplySecretSettingsForLlsAsync(RedisContentLocationStoreConfiguration configuration, AbsolutePath localCacheRoot)
{
(var secrets, var errors) = await _secretRetriever.TryRetrieveSecretsAsync();
if (secrets == null)
{
_logger.Error($"Unable to configure Local Location Store. {errors}");
return;
}
configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot);
if (_distributedSettings.IsMasterEligible)
{
// Use master selection by setting role to null
configuration.Checkpoint.Role = null;
}
else
{
// Not master eligible. Set role to worker.
configuration.Checkpoint.Role = Role.Worker;
}
var checkpointConfiguration = configuration.Checkpoint;
ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value);
ApplyIfNotNull(
_distributedSettings.HeartbeatIntervalMinutes,
value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.CreateCheckpointIntervalMinutes,
value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.RestoreCheckpointIntervalMinutes,
value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.SafeToLazilyUpdateMachineCountThreshold,
value => configuration.SafeToLazilyUpdateMachineCountThreshold = value);
configuration.EnableReconciliation = !_distributedSettings.Unsafe_DisableReconciliation;
configuration.ReconciliationCycleFrequency = TimeSpan.FromMinutes(_distributedSettings.ReconciliationCycleFrequencyMinutes);
configuration.ReconciliationMaxCycleSize = _distributedSettings.ReconciliationMaxCycleSize;
ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value);
ApplyIfNotNull(_distributedSettings.IncrementalCheckpointDegreeOfParallelism, value => configuration.Checkpoint.IncrementalCheckpointDegreeOfParallelism = value);
configuration.RedisGlobalStoreConnectionString = ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName)).Secret;
if (_distributedSettings.SecondaryGlobalRedisSecretName != null)
{
configuration.RedisGlobalStoreSecondaryConnectionString = ((PlainTextSecret)GetRequiredSecret(
secrets,
_distributedSettings.SecondaryGlobalRedisSecretName)).Secret;
}
ApplyIfNotNull(
_distributedSettings.ContentLocationReadMode,
value => configuration.ReadMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value));
ApplyIfNotNull(
_distributedSettings.ContentLocationWriteMode,
value => configuration.WriteMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value));
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value));
var errorBuilder = new StringBuilder();
var storageCredentials = GetStorageCredentials(secrets, errorBuilder);
Contract.Assert(storageCredentials != null && storageCredentials.Length > 0);
var blobStoreConfiguration = new BlobCentralStoreConfiguration(
credentials: storageCredentials,
containerName: "checkpoints",
checkpointsKey: "checkpoints-eventhub");
ApplyIfNotNull(
_distributedSettings.CentralStorageOperationTimeoutInMinutes,
value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value));
configuration.CentralStore = blobStoreConfiguration;
if (_distributedSettings.UseDistributedCentralStorage)
{
configuration.DistributedCentralStore = new DistributedCentralStoreConfiguration(localCacheRoot)
{
MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb,
PropagationDelay = TimeSpan.FromSeconds(
_distributedSettings.CentralStoragePropagationDelaySeconds),
PropagationIterations = _distributedSettings.CentralStoragePropagationIterations,
MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies
};
}
var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration(
eventHubName: "eventhub",
eventHubConnectionString: ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName)).Secret,
consumerGroupName: "$Default",
epoch: _keySpace + _distributedSettings.EventHubEpoch);
configuration.EventStore = eventStoreConfiguration;
ApplyIfNotNull(
_distributedSettings.MaxEventProcessingConcurrency,
value => eventStoreConfiguration.MaxEventProcessingConcurrency = value);
ApplyIfNotNull(
_distributedSettings.EventBatchSize,
value => eventStoreConfiguration.EventBatchSize = value);
ApplyIfNotNull(
_distributedSettings.EventProcessingMaxQueueSize,
value => eventStoreConfiguration.EventProcessingMaxQueueSize = value);
}
public RedisMemoizationStoreFactory CreateRedisCacheFactory(AbsolutePath localCacheRoot, out RedisContentLocationStoreConfiguration config)
{
var redisContentLocationStoreConfiguration = new RedisMemoizationStoreConfiguration
{
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
EvictionWindowSize = _distributedSettings.EvictionWindowSize,
EvictionPoolSize = _distributedSettings.EvictionPoolSize,
EvictionRemovalFraction = _distributedSettings.EvictionRemovalFraction,
MemoizationExpiryTime = TimeSpan.FromMinutes(_distributedSettings.RedisMemoizationExpiryTimeMinutes)
};
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineExpiryMinutes, v => redisContentLocationStoreConfiguration.MachineExpiry = TimeSpan.FromMinutes(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(localCacheRoot / "LocationDb")
{
StoreClusterState = _distributedSettings.StoreClusterStateInDatabase
};
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.GarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheEnabled, v => dbConfig.ContentCacheEnabled = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushDegreeOfParallelism, v => dbConfig.FlushDegreeOfParallelism = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushSingleTransaction, v => dbConfig.FlushSingleTransaction = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushPreservePercentInMemory, v => dbConfig.FlushPreservePercentInMemory = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheMaximumUpdatesPerFlush, v => dbConfig.CacheMaximumUpdatesPerFlush = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheFlushingMaximumInterval, v => dbConfig.CacheFlushingMaximumInterval = v);
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionIntervalMinutes,
v => dbConfig.FullRangeCompactionInterval = TimeSpan.FromMinutes(v));
redisContentLocationStoreConfiguration.Database = dbConfig;
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, localCacheRoot).GetAwaiter().GetResult();
}
if (_distributedSettings.IsRedisGarbageCollectionEnabled)
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = new RedisGarbageCollectionConfiguration()
{
MaximumEntryLastAccessTime = TimeSpan.FromMinutes(30)
};
}
else
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = null;
}
var contentHashBumpTime = TimeSpan.FromMinutes(_distributedSettings.ContentHashBumpTimeMinutes);
// RedisContentSecretName and RedisMachineLocationsSecretName can be null. HostConnectionStringProvider won't fail in this case.
IConnectionStringProvider contentConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisContentSecretName);
IConnectionStringProvider machineLocationsConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisMachineLocationsSecretName);
config = redisContentLocationStoreConfiguration;
return(new RedisMemoizationStoreFactory(
contentConnectionStringProvider,
machineLocationsConnectionStringProvider,
SystemClock.Instance,
contentHashBumpTime,
_keySpace,
configuration: redisContentLocationStoreConfiguration
));
}
protected override IContentStore CreateStore(
Context context,
TestFileCopier fileCopier,
DisposableDirectory testDirectory,
int index,
bool enableDistributedEviction,
int?replicaCreditInMinutes,
bool enableRepairHandling,
object additionalArgs)
{
var rootPath = testDirectory.Path / "Root";
var tempPath = testDirectory.Path / "Temp";
var configurationModel = new ConfigurationModel(Config);
var pathTransformer = new TestPathTransformer();
var localMachineData = pathTransformer.GetLocalMachineLocation(rootPath);
if (!_localDatabases.TryGetValue(context.Id, out var localDatabase))
{
localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
_localDatabases.TryAdd(context.Id, localDatabase);
}
if (!_localMachineDatabases.TryGetValue(context.Id, out var localMachineDatabase))
{
localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
_localMachineDatabases.TryAdd(context.Id, localMachineDatabase);
}
if (enableDistributedEviction && replicaCreditInMinutes == null)
{
// Apparently, replicaCreditInMinutes != null enables distributed eviction,
// so make sure replicaCreditInMinutes is set when enableDistributedEviction is
// true
replicaCreditInMinutes = 0;
}
_configuration = CreateContentLocationStoreConfiguration?.Invoke(rootPath, index) ?? new RedisContentLocationStoreConfiguration();
_configuration.BlobExpiryTimeMinutes = 10;
PostProcessConfiguration(_configuration, index);
var storeFactory = new MockRedisContentLocationStoreFactory(
localDatabase,
localMachineDatabase,
rootPath,
mockClock: TestClock,
_configuration);
var distributedContentStore = new DistributedContentStore <AbsolutePath>(
localMachineData,
(nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) =>
new FileSystemContentStore(
FileSystem,
TestClock,
rootPath,
configurationModel,
nagleQueue: nagleBlock,
distributedEvictionSettings: distributedEvictionSettings,
settings: contentStoreSettings,
trimBulkAsync: trimBulkAsync),
storeFactory,
fileCopier,
fileCopier,
pathTransformer,
fileCopier,
ContentAvailabilityGuarantee,
tempPath,
FileSystem,
RedisContentLocationStoreConstants.DefaultBatchSize,
settings: new DistributedContentStoreSettings
{
RetryIntervalForCopies = DistributedContentSessionTests.DefaultRetryIntervalsForTest,
PinConfiguration = PinConfiguration,
ShouldInlinePutBlob = true,
},
replicaCreditInMinutes: replicaCreditInMinutes,
clock: TestClock,
enableRepairHandling: enableRepairHandling,
contentStoreSettings: new ContentStoreSettings()
{
CheckFiles = true,
},
setPostInitializationCompletionAfterStartup: true);
distributedContentStore.DisposeContentStoreFactory = false;
return(distributedContentStore);
}
