private IMemoizationStore CreateServerSideLocalMemoizationStore(AbsolutePath path, DistributedContentStoreFactory factory)
{
var distributedSettings = _arguments.Configuration.DistributedContentSettings;
if (distributedSettings.UseRedisMetadataStore)
{
return(factory.CreateMemoizationStoreAsync().GetAwaiter().GetResult());
}
else if (distributedSettings.UseRoxisMetadataStore)
{
var config = new RoxisMemoizationDatabaseConfiguration();
ApplyIfNotNull(distributedSettings.RoxisMetadataStoreHost, v => config.MetadataClientConfiguration.GrpcHost = v);
ApplyIfNotNull(distributedSettings.RoxisMetadataStorePort, v => config.MetadataClientConfiguration.GrpcPort = v);
return(config.CreateStore(_logger, SystemClock.Instance));
}
else
{
var config = new RocksDbMemoizationStoreConfiguration()
{
Database = RocksDbContentLocationDatabaseConfiguration.FromDistributedContentSettings(
distributedSettings,
databasePath: path / "RocksDbMemoizationStore",
logsBackupPath: null,
logsKeepLongTerm: true),
};
config.Database.CleanOnInitialize = false;
config.Database.OnFailureDeleteExistingStoreAndRetry = true;
config.Database.MetadataGarbageCollectionEnabled = true;
return(new RocksDbMemoizationStore(_logger, SystemClock.Instance, config));
}
}
public IGrpcServiceEndpoint[] GetAdditionalEndpoints()
{
var contentMetadataFlags = (ContentMetadataStoreModeFlags)_distributedSettings.ContentMetadataStoreMode;
if ((contentMetadataFlags & ContentMetadataStoreModeFlags.Distributed) == 0)
{
return null;
}
var primaryCacheRoot = OrderedResolvedCacheSettings[0].ResolvedCacheRootPath;
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(primaryCacheRoot / "cms")
{
// Setting to false, until we have persistence for the db
CleanOnInitialize = false
};
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => dbConfig.GarbageCollectionInterval = TimeSpan.FromMinutes(v));
var service = new ContentMetadataService(new RocksDbContentMetadataStore(
_arguments.Overrides.Clock,
dbConfig));
return new IGrpcServiceEndpoint[]
{
service
};
}
public async Task DoesNotAllowWritingWhenInReadOnlyMode()
{
var configuration = new RocksDbContentLocationDatabaseConfiguration(_workingDirectory.Path)
{
CleanOnInitialize = false,
};
var context = new Context(Logger);
var ctx = new OperationContext(context);
// First, we create the database
{
var db = new RocksDbContentLocationDatabase(Clock, configuration, () => new MachineId[] { });
await db.StartupAsync(ctx).ShouldBeSuccess();
db.SetGlobalEntry("test", "hello");
await db.ShutdownAsync(ctx).ShouldBeSuccess();
}
configuration.OpenReadOnly = true;
{
var db = new RocksDbContentLocationDatabase(Clock, configuration, () => new MachineId[] { });
await db.StartupAsync(ctx).ShouldBeSuccess();
db.TryGetGlobalEntry("test", out var readValue);
readValue.Should().Be("hello");
Assert.Throws <BuildXLException>(() =>
{
db.SetGlobalEntry("test", "hello2");
});
await db.ShutdownAsync(ctx).ShouldBeSuccess();
}
}
private IMemoizationStore CreateServerSideLocalMemoizationStore(AbsolutePath path, DistributedContentStoreFactory factory)
{
var distributedSettings = _arguments.Configuration.DistributedContentSettings;
if (distributedSettings.UseRedisMetadataStore)
{
return(RedisMemoizationStoreFactory.CreateMemoizationStore(factory.Services.ContentLocationStoreServices.Instance));
}
else
{
var config = new RocksDbMemoizationStoreConfiguration()
{
Database = RocksDbContentLocationDatabaseConfiguration.FromDistributedContentSettings(
distributedSettings,
databasePath: path / "RocksDbMemoizationStore",
logsBackupPath: null,
logsKeepLongTerm: true),
};
config.Database.CleanOnInitialize = false;
config.Database.OnFailureDeleteExistingStoreAndRetry = true;
config.Database.MetadataGarbageCollectionEnabled = true;
return(new RocksDbMemoizationStore(_logger, SystemClock.Instance, config));
}
}
public void TestAbsolutePathSerialization()
{
var testPath = new AbsolutePath("M:/TESTPATH");
var testConfig = new RocksDbContentLocationDatabaseConfiguration(testPath);
string configString = ConfigurationPrinter.ConfigToString(testConfig);
configString.Should().NotContain("FileName:");
configString.Should().NotContain("Path:");
}
/// <nodoc />
public RocksDbContentPlacementPredictionStore(string storeLocation, bool clean)
{
_storeLocation = new AbsolutePath(storeLocation);
var dbLocation = _storeLocation / "db";
var config = new RocksDbContentLocationDatabaseConfiguration(dbLocation)
{
StoreClusterState = true,
CleanOnInitialize = clean,
GarbageCollectionInterval = Timeout.InfiniteTimeSpan
};
_clusterState = new ClusterState();
_database = new RocksDbContentLocationDatabase(SystemClock.Instance, config, () => new List <MachineId>());
}
public RedisMemoizationStoreFactory CreateRedisCacheFactory(AbsolutePath localCacheRoot, out RedisContentLocationStoreConfiguration config)
{
var redisContentLocationStoreConfiguration = new RedisMemoizationStoreConfiguration
{
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
EvictionWindowSize = _distributedSettings.EvictionWindowSize,
EvictionPoolSize = _distributedSettings.EvictionPoolSize,
EvictionRemovalFraction = _distributedSettings.EvictionRemovalFraction,
MemoizationExpiryTime = TimeSpan.FromMinutes(_distributedSettings.RedisMemoizationExpiryTimeMinutes)
};
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineExpiryMinutes, v => redisContentLocationStoreConfiguration.MachineExpiry = TimeSpan.FromMinutes(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(localCacheRoot / "LocationDb")
{
StoreClusterState = _distributedSettings.StoreClusterStateInDatabase
};
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.GarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheEnabled, v => dbConfig.ContentCacheEnabled = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushDegreeOfParallelism, v => dbConfig.FlushDegreeOfParallelism = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushSingleTransaction, v => dbConfig.FlushSingleTransaction = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushPreservePercentInMemory, v => dbConfig.FlushPreservePercentInMemory = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheMaximumUpdatesPerFlush, v => dbConfig.CacheMaximumUpdatesPerFlush = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheFlushingMaximumInterval, v => dbConfig.CacheFlushingMaximumInterval = v);
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionIntervalMinutes,
v => dbConfig.FullRangeCompactionInterval = TimeSpan.FromMinutes(v));
redisContentLocationStoreConfiguration.Database = dbConfig;
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, localCacheRoot).GetAwaiter().GetResult();
}
if (_distributedSettings.IsRedisGarbageCollectionEnabled)
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = new RedisGarbageCollectionConfiguration()
{
MaximumEntryLastAccessTime = TimeSpan.FromMinutes(30)
};
}
else
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = null;
}
var contentHashBumpTime = TimeSpan.FromMinutes(_distributedSettings.ContentHashBumpTimeMinutes);
// RedisContentSecretName and RedisMachineLocationsSecretName can be null. HostConnectionStringProvider won't fail in this case.
IConnectionStringProvider contentConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisContentSecretName);
IConnectionStringProvider machineLocationsConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisMachineLocationsSecretName);
config = redisContentLocationStoreConfiguration;
return(new RedisMemoizationStoreFactory(
contentConnectionStringProvider,
machineLocationsConnectionStringProvider,
SystemClock.Instance,
contentHashBumpTime,
_keySpace,
configuration: redisContentLocationStoreConfiguration
));
}
private async Task ApplySecretSettingsForLlsAsync(
RedisContentLocationStoreConfiguration configuration,
AbsolutePath localCacheRoot,
RocksDbContentLocationDatabaseConfiguration dbConfig)
{
(var secrets, var errors) = await _secretRetriever.TryRetrieveSecretsAsync();
if (secrets == null)
{
_logger.Error($"Unable to configure Local Location Store. {errors}");
return;
}
configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot);
if (_distributedSettings.IsMasterEligible)
{
// Use master selection by setting role to null
configuration.Checkpoint.Role = null;
}
else
{
// Not master eligible. Set role to worker.
configuration.Checkpoint.Role = Role.Worker;
}
// It is important to set the current role of the service, to have non-null Role column
// in all the tracing messages emitted to Kusto.
GlobalInfoStorage.SetServiceRole(configuration.Checkpoint.Role?.ToString() ?? "MasterEligible");
var checkpointConfiguration = configuration.Checkpoint;
ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value);
ApplyIfNotNull(
_distributedSettings.HeartbeatIntervalMinutes,
value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.CreateCheckpointIntervalMinutes,
value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.RestoreCheckpointIntervalMinutes,
value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.RestoreCheckpointTimeoutMinutes,
value => checkpointConfiguration.RestoreCheckpointTimeout = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.UpdateClusterStateIntervalSeconds,
value => checkpointConfiguration.UpdateClusterStateInterval = TimeSpan.FromSeconds(value));
ApplyIfNotNull(_distributedSettings.PacemakerEnabled, v => checkpointConfiguration.PacemakerEnabled = v);
ApplyIfNotNull(_distributedSettings.PacemakerNumberOfBuckets, v => checkpointConfiguration.PacemakerNumberOfBuckets = v);
ApplyIfNotNull(_distributedSettings.PacemakerUseRandomIdentifier, v => checkpointConfiguration.PacemakerUseRandomIdentifier = v);
ApplyIfNotNull(
_distributedSettings.SafeToLazilyUpdateMachineCountThreshold,
value => configuration.SafeToLazilyUpdateMachineCountThreshold = value);
configuration.EnableReconciliation = !_distributedSettings.Unsafe_DisableReconciliation;
configuration.ReconciliationCycleFrequency = TimeSpan.FromMinutes(_distributedSettings.ReconciliationCycleFrequencyMinutes);
configuration.ReconciliationMaxCycleSize = _distributedSettings.ReconciliationMaxCycleSize;
configuration.ReconciliationMaxRemoveHashesCycleSize = _distributedSettings.ReconciliationMaxRemoveHashesCycleSize;
configuration.ReconciliationMaxRemoveHashesAddPercentage = _distributedSettings.ReconciliationMaxRemoveHashesAddPercentage;
ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value);
ApplyIfNotNull(_distributedSettings.IncrementalCheckpointDegreeOfParallelism, value => configuration.Checkpoint.IncrementalCheckpointDegreeOfParallelism = value);
ApplyIfNotNull(_distributedSettings.UseRedisPreventThreadTheftFeature, value => configuration.UsePreventThreadTheftFeature = value);
configuration.RedisGlobalStoreConnectionString = ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName)).Secret;
if (_distributedSettings.SecondaryGlobalRedisSecretName != null)
{
configuration.RedisGlobalStoreSecondaryConnectionString = ((PlainTextSecret)GetRequiredSecret(
secrets,
_distributedSettings.SecondaryGlobalRedisSecretName)).Secret;
}
ApplyIfNotNull(_distributedSettings.RedisInternalLogSeverity, value =>
{
if (!Enum.TryParse <Severity>(value, out var parsedValue))
{
throw new ArgumentException($"Failed to parse `{nameof(_distributedSettings.RedisInternalLogSeverity)}` setting with value `{value}` into type `{nameof(Severity)}`");
}
configuration.RedisInternalLogSeverity = parsedValue;
});
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value));
ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => configuration.Checkpoint.RestoreCheckpointAgeThreshold = TimeSpan.FromMinutes(v));
// Need to disable cleaning database on initialization when restore checkpoint age is set.
ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => dbConfig.CleanOnInitialize = false);
var errorBuilder = new StringBuilder();
var storageCredentials = GetStorageCredentials(secrets, errorBuilder);
Contract.Assert(storageCredentials != null && storageCredentials.Length > 0);
var blobStoreConfiguration = new BlobCentralStoreConfiguration(
credentials: storageCredentials,
containerName: _arguments.HostInfo.AppendRingSpecifierIfNeeded("checkpoints", _distributedSettings.UseRingIsolation),
checkpointsKey: "checkpoints-eventhub");
ApplyIfNotNull(
_distributedSettings.CentralStorageOperationTimeoutInMinutes,
value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value));
configuration.CentralStore = blobStoreConfiguration;
if (_distributedSettings.UseDistributedCentralStorage)
{
var distributedCentralStoreConfiguration = new DistributedCentralStoreConfiguration(localCacheRoot)
{
MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb,
PropagationDelay = TimeSpan.FromSeconds(_distributedSettings.CentralStoragePropagationDelaySeconds),
PropagationIterations = _distributedSettings.CentralStoragePropagationIterations,
MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies
};
if (_distributedSettings.UseSelfCheckSettingsForDistributedCentralStorage)
{
distributedCentralStoreConfiguration.SelfCheckSettings = CreateSelfCheckSettings(_distributedSettings);
}
distributedCentralStoreConfiguration.TraceFileSystemContentStoreDiagnosticMessages = _distributedSettings.TraceFileSystemContentStoreDiagnosticMessages;
ApplyIfNotNull(_distributedSettings.DistributedCentralStoragePeerToPeerCopyTimeoutSeconds, v => distributedCentralStoreConfiguration.PeerToPeerCopyTimeout = TimeSpan.FromSeconds(v));
ApplyIfNotNull(_distributedSettings.DistributedCentralStorageImmutabilityOptimizations, v => distributedCentralStoreConfiguration.ImmutabilityOptimizations = v);
configuration.DistributedCentralStore = distributedCentralStoreConfiguration;
}
var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration(
eventHubName: _distributedSettings.EventHubName,
eventHubConnectionString: ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName)).Secret,
consumerGroupName: _distributedSettings.EventHubConsumerGroupName,
epoch: _keySpace + _distributedSettings.EventHubEpoch);
dbConfig.Epoch = eventStoreConfiguration.Epoch;
configuration.EventStore = eventStoreConfiguration;
ApplyIfNotNull(
_distributedSettings.MaxEventProcessingConcurrency,
value => eventStoreConfiguration.MaxEventProcessingConcurrency = value);
ApplyIfNotNull(
_distributedSettings.EventBatchSize,
value => eventStoreConfiguration.EventBatchSize = value);
ApplyIfNotNull(
_distributedSettings.EventProcessingMaxQueueSize,
value => eventStoreConfiguration.EventProcessingMaxQueueSize = value);
}
private RedisMemoizationStoreConfiguration CreateRedisConfiguration()
{
var primaryCacheRoot = OrderedResolvedCacheSettings[0].ResolvedCacheRootPath;
var redisContentLocationStoreConfiguration = new RedisMemoizationStoreConfiguration
{
Keyspace = _keySpace + RedisKeySpaceSalt,
LogReconciliationHashes = _distributedSettings.LogReconciliationHashes,
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
UseFullEvictionSort = _distributedSettings.UseFullEvictionSort,
EvictionWindowSize = _distributedSettings.EvictionWindowSize,
EvictionPoolSize = _distributedSettings.EvictionPoolSize,
UpdateStaleLocalLastAccessTimes = _distributedSettings.UpdateStaleLocalLastAccessTimes,
EvictionRemovalFraction = _distributedSettings.EvictionRemovalFraction,
EvictionDiscardFraction = _distributedSettings.EvictionDiscardFraction,
UseTieredDistributedEviction = _distributedSettings.UseTieredDistributedEviction,
MemoizationExpiryTime = TimeSpan.FromMinutes(_distributedSettings.RedisMemoizationExpiryTimeMinutes),
ProactiveCopyLocationsThreshold = _distributedSettings.ProactiveCopyLocationsThreshold,
UseBinManager = _distributedSettings.UseBinManager || _distributedSettings.ProactiveCopyUsePreferredLocations,
PreferredLocationsExpiryTime = TimeSpan.FromMinutes(_distributedSettings.PreferredLocationsExpiryTimeMinutes),
PrimaryMachineLocation = OrderedResolvedCacheSettings[0].MachineLocation,
AdditionalMachineLocations = OrderedResolvedCacheSettings.Skip(1).Select(r => r.MachineLocation).ToArray(),
MachineListPrioritizeDesignatedLocations = _distributedSettings.PrioritizeDesignatedLocationsOnCopies,
MachineListDeprioritizeMaster = _distributedSettings.DeprioritizeMasterOnCopies,
TouchContentHashLists = _distributedSettings.TouchContentHashLists
};
ApplyIfNotNull(_distributedSettings.ThrottledEvictionIntervalMinutes, v => redisContentLocationStoreConfiguration.ThrottledEvictionInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.RedisConnectionErrorLimit, v => redisContentLocationStoreConfiguration.RedisConnectionErrorLimit = v);
ApplyIfNotNull(_distributedSettings.RedisReconnectionLimitBeforeServiceRestart, v => redisContentLocationStoreConfiguration.RedisReconnectionLimitBeforeServiceRestart = v);
ApplyIfNotNull(_distributedSettings.TraceRedisFailures, v => redisContentLocationStoreConfiguration.TraceRedisFailures = v);
ApplyIfNotNull(_distributedSettings.TraceRedisTransientFailures, v => redisContentLocationStoreConfiguration.TraceRedisTransientFailures = v);
ApplyIfNotNull(_distributedSettings.MinRedisReconnectInterval, v => redisContentLocationStoreConfiguration.MinRedisReconnectInterval = v);
ApplyIfNotNull(_distributedSettings.CancelBatchWhenMultiplexerIsClosed, v => redisContentLocationStoreConfiguration.CancelBatchWhenMultiplexerIsClosed = v);
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineStateRecomputeIntervalMinutes, v => redisContentLocationStoreConfiguration.MachineStateRecomputeInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineActiveToClosedIntervalMinutes, v => redisContentLocationStoreConfiguration.MachineActiveToClosedInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineActiveToExpiredIntervalMinutes, v => redisContentLocationStoreConfiguration.MachineActiveToExpiredInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.TouchFrequencyMinutes, v => redisContentLocationStoreConfiguration.TouchFrequency = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.EvictionMinAgeMinutes, v => redisContentLocationStoreConfiguration.EvictionMinAge = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.RetryWindowSeconds, v => redisContentLocationStoreConfiguration.RetryWindow = TimeSpan.FromSeconds(v));
ApplyIfNotNull(_distributedSettings.RedisGetBlobTimeoutMilliseconds, v => redisContentLocationStoreConfiguration.BlobTimeout = TimeSpan.FromMilliseconds(v));
ApplyIfNotNull(_distributedSettings.RedisGetCheckpointStateTimeoutInSeconds, v => redisContentLocationStoreConfiguration.ClusterRedisOperationTimeout = TimeSpan.FromSeconds(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(primaryCacheRoot / "LocationDb")
{
LogsKeepLongTerm = true,
UseContextualEntryOperationLogging = _distributedSettings.UseContextualEntryDatabaseOperationLogging,
TraceTouches = _distributedSettings.TraceTouches,
};
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.GarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
if (_distributedSettings.EnableDistributedCache)
{
dbConfig.MetadataGarbageCollectionEnabled = true;
dbConfig.MetadataGarbageCollectionMaximumNumberOfEntriesToKeep = _distributedSettings.MaximumNumberOfMetadataEntriesToStore;
}
ApplyIfNotNull(
_distributedSettings.ContentLocationDatabaseOpenReadOnly,
v => dbConfig.OpenReadOnly = v && !_distributedSettings.IsMasterEligible);
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionIntervalMinutes,
v =>
{
if (v < 0)
{
throw new ArgumentException($"`{nameof(_distributedSettings.FullRangeCompactionIntervalMinutes)}` must be greater than or equal to 0");
}
if (v > 0)
{
dbConfig.FullRangeCompactionInterval = TimeSpan.FromMinutes(v);
}
});
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionVariant,
v =>
{
if (!Enum.TryParse <FullRangeCompactionVariant>(v, out var variant))
{
throw new ArgumentException($"Failed to parse `{nameof(_distributedSettings.FullRangeCompactionVariant)}` setting with value `{_distributedSettings.FullRangeCompactionVariant}` into type `{nameof(FullRangeCompactionVariant)}`");
}
dbConfig.FullRangeCompactionVariant = variant;
});
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionByteIncrementStep,
v => dbConfig.FullRangeCompactionByteIncrementStep = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseEnableDynamicLevelTargetSizes, v => dbConfig.EnableDynamicLevelTargetSizes = v);
if (_distributedSettings.ContentLocationDatabaseLogsBackupEnabled)
{
dbConfig.LogsBackupPath = primaryCacheRoot / "LocationDbLogs";
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseLogsBackupRetentionMinutes, v => dbConfig.LogsRetention = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseEnumerateSortedKeysFromStorageBufferSize, v => dbConfig.EnumerateSortedKeysFromStorageBufferSize = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseEnumerateEntriesWithSortedKeysFromStorageBufferSize, v => dbConfig.EnumerateEntriesWithSortedKeysFromStorageBufferSize = v);
redisContentLocationStoreConfiguration.Database = dbConfig;
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, primaryCacheRoot, dbConfig).GetAwaiter().GetResult();
}
_arguments.Overrides.Override(redisContentLocationStoreConfiguration);
ConfigurationPrinter.TraceConfiguration(redisContentLocationStoreConfiguration, _logger);
return(redisContentLocationStoreConfiguration);
}
public IContentStore CreateContentStore(
AbsolutePath localCacheRoot,
NagleQueue <ContentHash> evictionAnnouncer = null,
ProactiveReplicationArgs replicationSettings = null,
DistributedEvictionSettings distributedEvictionSettings = null,
bool checkLocalFiles = true,
TrimBulkAsync trimBulkAsync = null)
{
var redisContentLocationStoreConfiguration = new RedisContentLocationStoreConfiguration
{
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
EvictionWindowSize = _distributedSettings.EvictionWindowSize
};
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineExpiryMinutes, v => redisContentLocationStoreConfiguration.MachineExpiry = TimeSpan.FromMinutes(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(localCacheRoot / "LocationDb")
{
StoreClusterState = _distributedSettings.StoreClusterStateInDatabase
};
redisContentLocationStoreConfiguration.Database = dbConfig;
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.LocalDatabaseGarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheEnabled, v => dbConfig.CacheEnabled = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushDegreeOfParallelism, v => dbConfig.FlushDegreeOfParallelism = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushSingleTransaction, v => dbConfig.FlushSingleTransaction = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushPreservePercentInMemory, v => dbConfig.FlushPreservePercentInMemory = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheMaximumUpdatesPerFlush, v => dbConfig.CacheMaximumUpdatesPerFlush = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheFlushingMaximumInterval, v => dbConfig.CacheFlushingMaximumInterval = v);
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, localCacheRoot).GetAwaiter().GetResult();
}
if (_distributedSettings.IsRedisGarbageCollectionEnabled)
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = new RedisGarbageCollectionConfiguration()
{
MaximumEntryLastAccessTime = TimeSpan.FromMinutes(30)
};
}
else
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = null;
}
var localMachineLocation = _arguments.PathTransformer.GetLocalMachineLocation(localCacheRoot);
var contentHashBumpTime = TimeSpan.FromMinutes(_distributedSettings.ContentHashBumpTimeMinutes);
// RedisContentSecretName and RedisMachineLocationsSecretName can be null. HostConnectionStringProvider won't fail in this case.
IConnectionStringProvider contentConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisContentSecretName);
IConnectionStringProvider machineLocationsConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisMachineLocationsSecretName);
var redisContentLocationStoreFactory = new RedisContentLocationStoreFactory(
contentConnectionStringProvider,
machineLocationsConnectionStringProvider,
SystemClock.Instance,
contentHashBumpTime,
_keySpace,
localMachineLocation,
configuration: redisContentLocationStoreConfiguration
);
ReadOnlyDistributedContentSession <AbsolutePath> .ContentAvailabilityGuarantee contentAvailabilityGuarantee;
if (string.IsNullOrEmpty(_distributedSettings.ContentAvailabilityGuarantee))
{
contentAvailabilityGuarantee =
ReadOnlyDistributedContentSession <AbsolutePath> .ContentAvailabilityGuarantee
.FileRecordsExist;
}
else if (!Enum.TryParse(_distributedSettings.ContentAvailabilityGuarantee, true, out contentAvailabilityGuarantee))
{
throw new ArgumentException($"Unable to parse {nameof(_distributedSettings.ContentAvailabilityGuarantee)}: [{_distributedSettings.ContentAvailabilityGuarantee}]");
}
PinConfiguration pinConfiguration = null;
if (_distributedSettings.IsPinBetterEnabled)
{
pinConfiguration = new PinConfiguration();
if (_distributedSettings.PinRisk.HasValue)
{
pinConfiguration.PinRisk = _distributedSettings.PinRisk.Value;
}
if (_distributedSettings.MachineRisk.HasValue)
{
pinConfiguration.MachineRisk = _distributedSettings.MachineRisk.Value;
}
if (_distributedSettings.FileRisk.HasValue)
{
pinConfiguration.FileRisk = _distributedSettings.FileRisk.Value;
}
if (_distributedSettings.MaxIOOperations.HasValue)
{
pinConfiguration.MaxIOOperations = _distributedSettings.MaxIOOperations.Value;
}
pinConfiguration.UsePinCache = _distributedSettings.IsPinCachingEnabled;
if (_distributedSettings.PinCacheReplicaCreditRetentionMinutes.HasValue)
{
pinConfiguration.PinCacheReplicaCreditRetentionMinutes = _distributedSettings.PinCacheReplicaCreditRetentionMinutes.Value;
}
if (_distributedSettings.PinCacheReplicaCreditRetentionDecay.HasValue)
{
pinConfiguration.PinCacheReplicaCreditRetentionDecay = _distributedSettings.PinCacheReplicaCreditRetentionDecay.Value;
}
}
var lazyTouchContentHashBumpTime = _distributedSettings.IsTouchEnabled ? (TimeSpan?)contentHashBumpTime : null;
if (redisContentLocationStoreConfiguration.ReadMode == ContentLocationMode.LocalLocationStore)
{
// LocalLocationStore has its own internal notion of lazy touch/registration. We disable the lazy touch in distributed content store
// because it can conflict with behavior of the local location store.
lazyTouchContentHashBumpTime = null;
}
var contentStoreSettings = FromDistributedSettings(_distributedSettings);
ConfigurationModel configurationModel = null;
if (_arguments.Configuration.LocalCasSettings.CacheSettingsByCacheName.TryGetValue(_arguments.Configuration.LocalCasSettings.CasClientSettings.DefaultCacheName, out var namedCacheSettings))
{
configurationModel = new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota(namedCacheSettings.CacheSizeQuotaString)));
}
_logger.Debug("Creating a distributed content store for Autopilot");
var contentStore =
new DistributedContentStore <AbsolutePath>(
localMachineLocation,
(announcer, evictionSettings, checkLocal, trimBulk) =>
ContentStoreFactory.CreateContentStore(_fileSystem, localCacheRoot, announcer, distributedEvictionSettings: evictionSettings,
contentStoreSettings: contentStoreSettings, trimBulkAsync: trimBulk, configurationModel: configurationModel),
redisContentLocationStoreFactory,
_arguments.Copier,
_arguments.Copier,
_arguments.PathTransformer,
contentAvailabilityGuarantee,
localCacheRoot,
_fileSystem,
_distributedSettings.RedisBatchPageSize,
new DistributedContentStoreSettings()
{
UseTrustedHash = _distributedSettings.UseTrustedHash,
CleanRandomFilesAtRoot = _distributedSettings.CleanRandomFilesAtRoot,
TrustedHashFileSizeBoundary = _distributedSettings.TrustedHashFileSizeBoundary,
ParallelHashingFileSizeBoundary = _distributedSettings.ParallelHashingFileSizeBoundary,
MaxConcurrentCopyOperations = _distributedSettings.MaxConcurrentCopyOperations,
PinConfiguration = pinConfiguration,
EmptyFileHashShortcutEnabled = _distributedSettings.EmptyFileHashShortcutEnabled,
RetryIntervalForCopies = _distributedSettings.RetryIntervalForCopies,
},
replicaCreditInMinutes: _distributedSettings.IsDistributedEvictionEnabled?_distributedSettings.ReplicaCreditInMinutes: null,
enableRepairHandling: _distributedSettings.IsRepairHandlingEnabled,
contentHashBumpTime: lazyTouchContentHashBumpTime,
contentStoreSettings: contentStoreSettings);
_logger.Debug("Created Distributed content store.");
return(contentStore);
}
private RedisMemoizationStoreConfiguration CreateRedisConfiguration()
{
var primaryCacheRoot = OrderedResolvedCacheSettings[0].ResolvedCacheRootPath;
var redisContentLocationStoreConfiguration = new RedisMemoizationStoreConfiguration
{
Keyspace = _keySpace + RedisKeySpaceSalt,
LogReconciliationHashes = _distributedSettings.LogReconciliationHashes,
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
UseFullEvictionSort = _distributedSettings.UseFullEvictionSort,
EvictionWindowSize = _distributedSettings.EvictionWindowSize,
EvictionPoolSize = _distributedSettings.EvictionPoolSize,
UpdateStaleLocalLastAccessTimes = _distributedSettings.UpdateStaleLocalLastAccessTimes,
EvictionRemovalFraction = _distributedSettings.EvictionRemovalFraction,
EvictionDiscardFraction = _distributedSettings.EvictionDiscardFraction,
UseTieredDistributedEviction = _distributedSettings.UseTieredDistributedEviction,
MemoizationExpiryTime = TimeSpan.FromMinutes(_distributedSettings.RedisMemoizationExpiryTimeMinutes),
ProactiveCopyLocationsThreshold = _distributedSettings.ProactiveCopyLocationsThreshold,
UseBinManager = _distributedSettings.UseBinManager || _distributedSettings.ProactiveCopyUsePreferredLocations,
PreferredLocationsExpiryTime = TimeSpan.FromMinutes(_distributedSettings.PreferredLocationsExpiryTimeMinutes),
PrimaryMachineLocation = OrderedResolvedCacheSettings[0].MachineLocation,
MachineListPrioritizeDesignatedLocations = _distributedSettings.PrioritizeDesignatedLocationsOnCopies,
MachineListDeprioritizeMaster = _distributedSettings.DeprioritizeMasterOnCopies,
TouchContentHashLists = _distributedSettings.TouchContentHashLists,
};
ApplyIfNotNull(_distributedSettings.BlobOperationLimitCount, v => redisContentLocationStoreConfiguration.BlobOperationLimitCount = v);
ApplyIfNotNull(_distributedSettings.BlobOperationLimitSpanSeconds, v => redisContentLocationStoreConfiguration.BlobOperationLimitSpan = TimeSpan.FromSeconds(v));
ApplyIfNotNull(_distributedSettings.UseSeparateConnectionForRedisBlobs, v => redisContentLocationStoreConfiguration.UseSeparateConnectionForRedisBlobs = v);
// Stop passing additional stores when fully transitioned to unified mode
// since all drives appear under the same machine id
if (_distributedSettings.GetMultiplexMode() != MultiplexMode.Unified)
{
redisContentLocationStoreConfiguration.AdditionalMachineLocations = OrderedResolvedCacheSettings.Skip(1).Select(r => r.MachineLocation).ToArray();
}
ApplyIfNotNull(_distributedSettings.ThrottledEvictionIntervalMinutes, v => redisContentLocationStoreConfiguration.ThrottledEvictionInterval = TimeSpan.FromMinutes(v));
// Redis-related configuration.
ApplyIfNotNull(_distributedSettings.RedisConnectionErrorLimit, v => redisContentLocationStoreConfiguration.RedisConnectionErrorLimit = v);
ApplyIfNotNull(_distributedSettings.RedisReconnectionLimitBeforeServiceRestart, v => redisContentLocationStoreConfiguration.RedisReconnectionLimitBeforeServiceRestart = v);
ApplyIfNotNull(_distributedSettings.DefaultRedisOperationTimeoutInSeconds, v => redisContentLocationStoreConfiguration.OperationTimeout = TimeSpan.FromSeconds(v));
ApplyIfNotNull(_distributedSettings.MinRedisReconnectInterval, v => redisContentLocationStoreConfiguration.MinRedisReconnectInterval = v);
ApplyIfNotNull(_distributedSettings.CancelBatchWhenMultiplexerIsClosed, v => redisContentLocationStoreConfiguration.CancelBatchWhenMultiplexerIsClosed = v);
if (_distributedSettings.RedisExponentialBackoffRetryCount != null)
{
var exponentialBackoffConfiguration = new ExponentialBackoffConfiguration(
_distributedSettings.RedisExponentialBackoffRetryCount.Value,
minBackoff: IfNotNull(_distributedSettings.RedisExponentialBackoffMinIntervalInSeconds, v => TimeSpan.FromSeconds(v)),
maxBackoff: IfNotNull(_distributedSettings.RedisExponentialBackoffMaxIntervalInSeconds, v => TimeSpan.FromSeconds(v)),
deltaBackoff: IfNotNull(_distributedSettings.RedisExponentialBackoffDeltaIntervalInSeconds, v => TimeSpan.FromSeconds(v))
);
redisContentLocationStoreConfiguration.ExponentialBackoffConfiguration = exponentialBackoffConfiguration;
}
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineStateRecomputeIntervalMinutes, v => redisContentLocationStoreConfiguration.MachineStateRecomputeInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineActiveToClosedIntervalMinutes, v => redisContentLocationStoreConfiguration.MachineActiveToClosedInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineActiveToExpiredIntervalMinutes, v => redisContentLocationStoreConfiguration.MachineActiveToExpiredInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.TouchFrequencyMinutes, v => redisContentLocationStoreConfiguration.TouchFrequency = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.ReconcileCacheLifetimeMinutes, v => redisContentLocationStoreConfiguration.ReconcileCacheLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.EvictionMinAgeMinutes, v => redisContentLocationStoreConfiguration.EvictionMinAge = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.RetryWindowSeconds, v => redisContentLocationStoreConfiguration.RetryWindow = TimeSpan.FromSeconds(v));
ApplyIfNotNull(_distributedSettings.Unsafe_MasterThroughputCheckMode, v => redisContentLocationStoreConfiguration.MasterThroughputCheckMode = v);
ApplyIfNotNull(_distributedSettings.Unsafe_EventHubCursorPosition, v => redisContentLocationStoreConfiguration.EventHubCursorPosition = v);
ApplyIfNotNull(_distributedSettings.RedisGetBlobTimeoutMilliseconds, v => redisContentLocationStoreConfiguration.BlobTimeout = TimeSpan.FromMilliseconds(v));
ApplyIfNotNull(_distributedSettings.RedisGetCheckpointStateTimeoutInSeconds, v => redisContentLocationStoreConfiguration.ClusterRedisOperationTimeout = TimeSpan.FromSeconds(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = RocksDbContentLocationDatabaseConfiguration.FromDistributedContentSettings(
_distributedSettings,
primaryCacheRoot / "LocationDb",
primaryCacheRoot / "LocationDbLogs",
logsKeepLongTerm: true
);
redisContentLocationStoreConfiguration.Database = dbConfig;
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, primaryCacheRoot, dbConfig).GetAwaiter().GetResult();
}
_arguments.Overrides.Override(redisContentLocationStoreConfiguration);
ConfigurationPrinter.TraceConfiguration(redisContentLocationStoreConfiguration, _logger);
return(redisContentLocationStoreConfiguration);
}
private async Task ApplySecretSettingsForLlsAsync(RedisContentLocationStoreConfiguration configuration, AbsolutePath localCacheRoot, RocksDbContentLocationDatabaseConfiguration dbConfig)
{
(var secrets, var errors) = await _secretRetriever.TryRetrieveSecretsAsync();
if (secrets == null)
{
_logger.Error($"Unable to configure Local Location Store. {errors}");
return;
}
configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot);
if (_distributedSettings.IsMasterEligible)
{
// Use master selection by setting role to null
configuration.Checkpoint.Role = null;
}
else
{
// Not master eligible. Set role to worker.
configuration.Checkpoint.Role = Role.Worker;
}
var checkpointConfiguration = configuration.Checkpoint;
ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value);
ApplyIfNotNull(
_distributedSettings.HeartbeatIntervalMinutes,
value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.CreateCheckpointIntervalMinutes,
value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.RestoreCheckpointIntervalMinutes,
value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.SafeToLazilyUpdateMachineCountThreshold,
value => configuration.SafeToLazilyUpdateMachineCountThreshold = value);
configuration.EnableReconciliation = !_distributedSettings.Unsafe_DisableReconciliation;
configuration.ReconciliationCycleFrequency = TimeSpan.FromMinutes(_distributedSettings.ReconciliationCycleFrequencyMinutes);
configuration.ReconciliationMaxCycleSize = _distributedSettings.ReconciliationMaxCycleSize;
ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value);
ApplyIfNotNull(_distributedSettings.IncrementalCheckpointDegreeOfParallelism, value => configuration.Checkpoint.IncrementalCheckpointDegreeOfParallelism = value);
configuration.RedisGlobalStoreConnectionString = ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName)).Secret;
if (_distributedSettings.SecondaryGlobalRedisSecretName != null)
{
configuration.RedisGlobalStoreSecondaryConnectionString = ((PlainTextSecret)GetRequiredSecret(
secrets,
_distributedSettings.SecondaryGlobalRedisSecretName)).Secret;
}
ApplyIfNotNull(
_distributedSettings.ContentLocationReadMode,
value => configuration.ReadMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value));
ApplyIfNotNull(
_distributedSettings.ContentLocationWriteMode,
value => configuration.WriteMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value));
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value));
ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => configuration.Checkpoint.RestoreCheckpointAgeThreshold = TimeSpan.FromMinutes(v));
// Need to disable cleaning database on initialization when restore checkpoint age is set.
ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => dbConfig.CleanOnInitialize = false);
var errorBuilder = new StringBuilder();
var storageCredentials = GetStorageCredentials(secrets, errorBuilder);
Contract.Assert(storageCredentials != null && storageCredentials.Length > 0);
var blobStoreConfiguration = new BlobCentralStoreConfiguration(
credentials: storageCredentials,
containerName: "checkpoints",
checkpointsKey: "checkpoints-eventhub");
ApplyIfNotNull(
_distributedSettings.CentralStorageOperationTimeoutInMinutes,
value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value));
configuration.CentralStore = blobStoreConfiguration;
if (_distributedSettings.UseDistributedCentralStorage)
{
var distributedCentralStoreConfiguration = new DistributedCentralStoreConfiguration(localCacheRoot)
{
MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb,
PropagationDelay = TimeSpan.FromSeconds(
_distributedSettings.CentralStoragePropagationDelaySeconds),
PropagationIterations = _distributedSettings.CentralStoragePropagationIterations,
MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies
};
ApplyIfNotNull(_distributedSettings.DistributedCentralStoragePeerToPeerCopyTimeoutSeconds, v => distributedCentralStoreConfiguration.PeerToPeerCopyTimeout = TimeSpan.FromSeconds(v));
configuration.DistributedCentralStore = distributedCentralStoreConfiguration;
}
var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration(
eventHubName: "eventhub",
eventHubConnectionString: ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName)).Secret,
consumerGroupName: "$Default",
epoch: _keySpace + _distributedSettings.EventHubEpoch);
configuration.EventStore = eventStoreConfiguration;
ApplyIfNotNull(
_distributedSettings.MaxEventProcessingConcurrency,
value => eventStoreConfiguration.MaxEventProcessingConcurrency = value);
ApplyIfNotNull(
_distributedSettings.EventBatchSize,
value => eventStoreConfiguration.EventBatchSize = value);
ApplyIfNotNull(
_distributedSettings.EventProcessingMaxQueueSize,
value => eventStoreConfiguration.EventProcessingMaxQueueSize = value);
}
private RedisMemoizationStoreConfiguration CreateRedisConfiguration()
{
var primaryCacheRoot = OrderedResolvedCacheSettings[0].ResolvedCacheRootPath;
var redisContentLocationStoreConfiguration = new RedisMemoizationStoreConfiguration
{
Keyspace = _keySpace + RedisKeySpaceSalt,
LogReconciliationHashes = _distributedSettings.LogReconciliationHashes,
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
UseFullEvictionSort = _distributedSettings.UseFullEvictionSort,
EvictionWindowSize = _distributedSettings.EvictionWindowSize,
EvictionPoolSize = _distributedSettings.EvictionPoolSize,
UpdateStaleLocalLastAccessTimes = _distributedSettings.UpdateStaleLocalLastAccessTimes,
EvictionRemovalFraction = _distributedSettings.EvictionRemovalFraction,
EvictionDiscardFraction = _distributedSettings.EvictionDiscardFraction,
UseTieredDistributedEviction = _distributedSettings.UseTieredDistributedEviction,
MemoizationExpiryTime = TimeSpan.FromMinutes(_distributedSettings.RedisMemoizationExpiryTimeMinutes),
ProactiveCopyLocationsThreshold = _distributedSettings.ProactiveCopyLocationsThreshold,
UseBinManager = _distributedSettings.UseBinManager || _distributedSettings.ProactiveCopyUsePreferredLocations,
PreferredLocationsExpiryTime = TimeSpan.FromMinutes(_distributedSettings.PreferredLocationsExpiryTimeMinutes),
PrimaryMachineLocation = OrderedResolvedCacheSettings[0].MachineLocation,
AdditionalMachineLocations = OrderedResolvedCacheSettings.Skip(1).Select(r => r.MachineLocation).ToArray()
};
ApplyIfNotNull(_distributedSettings.RedisConnectionErrorLimit, v => redisContentLocationStoreConfiguration.RedisConnectionErrorLimit = v);
ApplyIfNotNull(_distributedSettings.TraceRedisFailures, v => redisContentLocationStoreConfiguration.TraceRedisFailures = v);
ApplyIfNotNull(_distributedSettings.TraceRedisTransientFailures, v => redisContentLocationStoreConfiguration.TraceRedisTransientFailures = v);
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineExpiryMinutes, v => redisContentLocationStoreConfiguration.MachineExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.TouchFrequencyMinutes, v => redisContentLocationStoreConfiguration.TouchFrequency = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.EvictionMinAgeMinutes, v => redisContentLocationStoreConfiguration.EvictionMinAge = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.RetryWindowSeconds, v => redisContentLocationStoreConfiguration.RetryWindow = TimeSpan.FromSeconds(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(primaryCacheRoot / "LocationDb")
{
StoreClusterState = _distributedSettings.StoreClusterStateInDatabase,
LogsKeepLongTerm = true,
UseContextualEntryOperationLogging = _distributedSettings.UseContextualEntryDatabaseOperationLogging
};
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.GarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
if (_distributedSettings.EnableDistributedCache)
{
dbConfig.MetadataGarbageCollectionEnabled = true;
dbConfig.MetadataGarbageCollectionMaximumNumberOfEntriesToKeep = _distributedSettings.MaximumNumberOfMetadataEntriesToStore;
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheEnabled, v => dbConfig.ContentCacheEnabled = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushDegreeOfParallelism, v => dbConfig.FlushDegreeOfParallelism = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushTransactionSize, v => dbConfig.FlushTransactionSize = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushSingleTransaction, v => dbConfig.FlushSingleTransaction = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushPreservePercentInMemory, v => dbConfig.FlushPreservePercentInMemory = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheMaximumUpdatesPerFlush, v => dbConfig.CacheMaximumUpdatesPerFlush = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheFlushingMaximumInterval, v => dbConfig.CacheFlushingMaximumInterval = v);
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionIntervalMinutes,
v => dbConfig.FullRangeCompactionInterval = TimeSpan.FromMinutes(v));
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionVariant,
v => {
if (Enum.TryParse <FullRangeCompactionVariant>(v, out var variant))
{
dbConfig.FullRangeCompactionVariant = variant;
}
else
{
throw new ArgumentException($"Failed to parse `{nameof(_distributedSettings.FullRangeCompactionVariant)}` setting with value `{_distributedSettings.FullRangeCompactionVariant}` into type `{nameof(FullRangeCompactionVariant)}`");
}
});
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionByteIncrementStep,
v => dbConfig.FullRangeCompactionByteIncrementStep = v);
if (_distributedSettings.ContentLocationDatabaseLogsBackupEnabled)
{
dbConfig.LogsBackupPath = primaryCacheRoot / "LocationDbLogs";
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseLogsBackupRetentionMinutes, v => dbConfig.LogsRetention = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseEnumerateSortedKeysFromStorageBufferSize, v => dbConfig.EnumerateSortedKeysFromStorageBufferSize = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseEnumerateEntriesWithSortedKeysFromStorageBufferSize, v => dbConfig.EnumerateEntriesWithSortedKeysFromStorageBufferSize = v);
redisContentLocationStoreConfiguration.Database = dbConfig;
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, primaryCacheRoot, dbConfig).GetAwaiter().GetResult();
}
if (_distributedSettings.IsRedisGarbageCollectionEnabled)
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = new RedisGarbageCollectionConfiguration()
{
MaximumEntryLastAccessTime = TimeSpan.FromMinutes(30)
};
}
else
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = null;
}
_arguments.Overrides.Override(redisContentLocationStoreConfiguration);
ConfigurationPrinter.TraceConfiguration(redisContentLocationStoreConfiguration, _logger);
return(redisContentLocationStoreConfiguration);
}
public RedisMemoizationStoreFactory CreateRedisCacheFactory(AbsolutePath localCacheRoot, out RedisContentLocationStoreConfiguration config)
{
var redisContentLocationStoreConfiguration = new RedisMemoizationStoreConfiguration
{
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize,
EvictionWindowSize = _distributedSettings.EvictionWindowSize,
EvictionPoolSize = _distributedSettings.EvictionPoolSize,
EvictionRemovalFraction = _distributedSettings.EvictionRemovalFraction,
EvictionDiscardFraction = _distributedSettings.EvictionDiscardFraction,
UseTieredDistributedEviction = _distributedSettings.UseTieredDistributedEviction,
MemoizationExpiryTime = TimeSpan.FromMinutes(_distributedSettings.RedisMemoizationExpiryTimeMinutes),
ProactiveCopyLocationsThreshold = _distributedSettings.ProactiveCopyLocationsThreshold,
PreferredLocationsExpiryTime = TimeSpan.FromMinutes(_distributedSettings.PreferredLocationsExpiryTimeMinutes),
DelayForProactiveReplication = TimeSpan.FromSeconds(_distributedSettings.ProactiveReplicationDelaySeconds),
ProactiveReplicationCopyLimit = _distributedSettings.ProactiveReplicationCopyLimit,
EnableProactiveReplication = _distributedSettings.EnableProactiveReplication,
};
ApplyIfNotNull(_distributedSettings.RedisConnectionErrorLimit, v => redisContentLocationStoreConfiguration.RedisConnectionErrorLimit = v);
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ContentLifetime = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineRisk, v => redisContentLocationStoreConfiguration.MachineRisk = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineExpiryMinutes, v => redisContentLocationStoreConfiguration.MachineExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.TouchFrequencyMinutes, v => redisContentLocationStoreConfiguration.TouchFrequency = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.EvictionMinAgeMinutes, v => redisContentLocationStoreConfiguration.EvictionMinAge = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.RetryWindowSeconds, v => redisContentLocationStoreConfiguration.RetryWindow = TimeSpan.FromSeconds(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(localCacheRoot / "LocationDb")
{
StoreClusterState = _distributedSettings.StoreClusterStateInDatabase,
LogsKeepLongTerm = true,
};
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.GarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
if (_distributedSettings.EnableDistributedCache)
{
dbConfig.MetadataGarbageCollectionEnabled = true;
dbConfig.MetadataGarbageCollectionMaximumNumberOfEntriesToKeep = _distributedSettings.MaximumNumberOfMetadataEntriesToStore;
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheEnabled, v => dbConfig.ContentCacheEnabled = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushDegreeOfParallelism, v => dbConfig.FlushDegreeOfParallelism = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushTransactionSize, v => dbConfig.FlushTransactionSize = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushSingleTransaction, v => dbConfig.FlushSingleTransaction = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseFlushPreservePercentInMemory, v => dbConfig.FlushPreservePercentInMemory = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheMaximumUpdatesPerFlush, v => dbConfig.CacheMaximumUpdatesPerFlush = v);
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseCacheFlushingMaximumInterval, v => dbConfig.CacheFlushingMaximumInterval = v);
ApplyIfNotNull(
_distributedSettings.FullRangeCompactionIntervalMinutes,
v => dbConfig.FullRangeCompactionInterval = TimeSpan.FromMinutes(v));
if (_distributedSettings.ContentLocationDatabaseLogsBackupEnabled)
{
dbConfig.LogsBackupPath = localCacheRoot / "LocationDbLogs";
}
ApplyIfNotNull(_distributedSettings.ContentLocationDatabaseLogsBackupRetentionMinutes, v => dbConfig.LogsRetention = TimeSpan.FromMinutes(v));
redisContentLocationStoreConfiguration.Database = dbConfig;
ApplySecretSettingsForLlsAsync(redisContentLocationStoreConfiguration, localCacheRoot, dbConfig).GetAwaiter().GetResult();
}
if (_distributedSettings.IsRedisGarbageCollectionEnabled)
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = new RedisGarbageCollectionConfiguration()
{
MaximumEntryLastAccessTime = TimeSpan.FromMinutes(30)
};
}
else
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = null;
}
var contentHashBumpTime = TimeSpan.FromMinutes(_distributedSettings.ContentHashBumpTimeMinutes);
// RedisContentSecretName and RedisMachineLocationsSecretName can be null. HostConnectionStringProvider won't fail in this case.
IConnectionStringProvider contentConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisContentSecretName);
IConnectionStringProvider machineLocationsConnectionStringProvider = TryCreateRedisConnectionStringProvider(_redisContentSecretNames.RedisMachineLocationsSecretName);
_arguments.Overrides.Override(redisContentLocationStoreConfiguration);
config = redisContentLocationStoreConfiguration;
return(new RedisMemoizationStoreFactory(
contentConnectionStringProvider,
machineLocationsConnectionStringProvider,
_arguments.Overrides.Clock,
contentHashBumpTime,
_keySpace,
configuration: redisContentLocationStoreConfiguration
));
}
public async Task TestGarbageCollect()
{
var configuration = new RocksDbContentLocationDatabaseConfiguration(_workingDirectory.Path)
{
CleanOnInitialize = false,
};
var context = new Context(Logger);
var ctx = new OperationContext(context);
var keys = Enumerable.Range(0, 10).Select(i => (ShortHash)ContentHash.Random()).ToArray();
void setBlob(RocksDbContentMetadataDatabase db, ShortHash key)
{
db.PutBlob(key, key.ToByteArray());
}
KeyCheckResult checkBlob(RocksDbContentMetadataDatabase db, ShortHash key)
{
if (db.TryGetBlob(key, out var blob))
{
if (ByteArrayComparer.ArraysEqual(blob, key.ToByteArray()))
{
return(KeyCheckResult.Valid);
}
else
{
return(KeyCheckResult.Different);
}
}
else
{
return(KeyCheckResult.Missing);
}
}
{
var db = new RocksDbContentMetadataDatabase(Clock, configuration);
await db.StartupAsync(ctx).ShouldBeSuccess();
db.SetGlobalEntry("test", "hello");
setBlob(db, keys[0]);
checkBlob(db, keys[0]).Should().Be(KeyCheckResult.Valid);
await db.GarbageCollectAsync(ctx).ShouldBeSuccess();
setBlob(db, keys[1]);
checkBlob(db, keys[0]).Should().Be(KeyCheckResult.Valid);
checkBlob(db, keys[1]).Should().Be(KeyCheckResult.Valid);
await db.GarbageCollectAsync(ctx).ShouldBeSuccess();
checkBlob(db, keys[0]).Should().Be(KeyCheckResult.Missing);
checkBlob(db, keys[1]).Should().Be(KeyCheckResult.Valid);
await db.ShutdownAsync(ctx).ShouldBeSuccess();
}
{
var db = new RocksDbContentMetadataDatabase(Clock, configuration);
await db.StartupAsync(ctx).ShouldBeSuccess();
db.TryGetGlobalEntry("test", out var readValue);
readValue.Should().Be("hello");
setBlob(db, keys[2]);
checkBlob(db, keys[0]).Should().Be(KeyCheckResult.Missing);
checkBlob(db, keys[1]).Should().Be(KeyCheckResult.Valid);
checkBlob(db, keys[2]).Should().Be(KeyCheckResult.Valid);
await db.GarbageCollectAsync(ctx).ShouldBeSuccess();
checkBlob(db, keys[0]).Should().Be(KeyCheckResult.Missing);
checkBlob(db, keys[1]).Should().Be(KeyCheckResult.Missing);
checkBlob(db, keys[2]).Should().Be(KeyCheckResult.Valid);
await db.ShutdownAsync(ctx).ShouldBeSuccess();
}
}