public DistributedCentralStorage( DistributedCentralStoreConfiguration configuration, ILocationStore locationStore, IDistributedContentCopier copier, CentralStorage fallbackStorage) { _configuration = configuration; _copier = copier; _fallbackStorage = fallbackStorage; _locationStore = locationStore; var maxRetentionMb = configuration.MaxRetentionGb * 1024; var softRetentionMb = (int)(maxRetentionMb * 0.8); var cacheFolder = configuration.CacheRoot / CacheSubFolderName; _copierWorkingDirectory = new DisposableDirectory(copier.FileSystem, cacheFolder / "Temp"); // Create a private CAS for storing checkpoint data // Avoid introducing churn into primary CAS _privateCas = new FileSystemContentStoreInternal( copier.FileSystem, SystemClock.Instance, cacheFolder, new ConfigurationModel( new ContentStoreConfiguration(new MaxSizeQuota(hardExpression: maxRetentionMb + "MB", softExpression: softRetentionMb + "MB")), ConfigurationSelection.RequireAndUseInProcessConfiguration), settings: new ContentStoreSettings() { TraceFileSystemContentStoreDiagnosticMessages = _configuration.TraceFileSystemContentStoreDiagnosticMessages, SelfCheckSettings = _configuration.SelfCheckSettings, }); }
public CachingCentralStorage( DistributedCentralStoreConfiguration configuration, CentralStorage fallbackStorage, IAbsFileSystem fileSystem) { Configuration = configuration; _fallbackStorage = fallbackStorage; var maxRetentionMb = (int)Math.Ceiling(configuration.MaxRetentionGb * 1024); var softRetentionMb = (int)(maxRetentionMb * 0.8); var cacheFolder = configuration.CacheRoot / CacheSubFolderName; // Create a private CAS for storing checkpoint data // Avoid introducing churn into primary CAS PrivateCas = new FileSystemContentStoreInternal( fileSystem, SystemClock.Instance, cacheFolder, new ConfigurationModel( new ContentStoreConfiguration(new MaxSizeQuota(hardExpression: maxRetentionMb + "MB", softExpression: softRetentionMb + "MB")), ConfigurationSelection.RequireAndUseInProcessConfiguration), settings: new ContentStoreSettings() { TraceFileSystemContentStoreDiagnosticMessages = Configuration.TraceFileSystemContentStoreDiagnosticMessages, SelfCheckSettings = Configuration.SelfCheckSettings, }); }
public DistributedCentralStorage( DistributedCentralStoreConfiguration configuration, ILocationStore locationStore, DistributedContentCopier copier, CentralStorage fallbackStorage, IClock clock) : base(configuration, fallbackStorage, copier.FileSystem) { _copier = copier; _locationStore = locationStore; _checkpointCopies = new VolatileMap <ShortHash, CopyOperation>(clock); _checkpointStore = configuration.IsCheckpointAware ? _locationStore as ICheckpointStore : null; _copierWorkingDirectory = new DisposableDirectory(copier.FileSystem, PrivateCas !.RootPath / "Temp"); }
/// <nodoc /> public DistributedCentralStorage( DistributedCentralStoreConfiguration configuration, IDistributedContentCopier copier, CentralStorage fallbackStorage) { _configuration = configuration; _copier = copier; _fallbackStorage = fallbackStorage; var maxRetentionMb = configuration.MaxRetentionGb * 1024; var softRetentionMb = (int)(maxRetentionMb * 0.8); // Create a private CAS for storing checkpoint data // Avoid introducing churn into primary CAS _privateCas = new FileSystemContentStoreInternal( new PassThroughFileSystem(), SystemClock.Instance, configuration.CacheRoot / CacheSubFolderName, new ConfigurationModel( new ContentStoreConfiguration(new MaxSizeQuota(hardExpression: maxRetentionMb + "MB", softExpression: softRetentionMb + "MB")), ConfigurationSelection.RequireAndUseInProcessConfiguration)); }
private async Task ApplySecretSettingsForLlsAsync( RedisContentLocationStoreConfiguration configuration, AbsolutePath localCacheRoot, RocksDbContentLocationDatabaseConfiguration dbConfig) { (var secrets, var errors) = await _secretRetriever.TryRetrieveSecretsAsync(); if (secrets == null) { _logger.Error($"Unable to configure Local Location Store. {errors}"); return; } configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot); if (_distributedSettings.IsMasterEligible) { // Use master selection by setting role to null configuration.Checkpoint.Role = null; } else { // Not master eligible. Set role to worker. configuration.Checkpoint.Role = Role.Worker; } // It is important to set the current role of the service, to have non-null Role column // in all the tracing messages emitted to Kusto. GlobalInfoStorage.SetServiceRole(configuration.Checkpoint.Role?.ToString() ?? "MasterEligible"); var checkpointConfiguration = configuration.Checkpoint; ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value); ApplyIfNotNull( _distributedSettings.HeartbeatIntervalMinutes, value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.CreateCheckpointIntervalMinutes, value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.RestoreCheckpointIntervalMinutes, value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.RestoreCheckpointTimeoutMinutes, value => checkpointConfiguration.RestoreCheckpointTimeout = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.UpdateClusterStateIntervalSeconds, value => checkpointConfiguration.UpdateClusterStateInterval = TimeSpan.FromSeconds(value)); ApplyIfNotNull(_distributedSettings.PacemakerEnabled, v => checkpointConfiguration.PacemakerEnabled = v); ApplyIfNotNull(_distributedSettings.PacemakerNumberOfBuckets, v => checkpointConfiguration.PacemakerNumberOfBuckets = v); ApplyIfNotNull(_distributedSettings.PacemakerUseRandomIdentifier, v => checkpointConfiguration.PacemakerUseRandomIdentifier = v); ApplyIfNotNull( _distributedSettings.SafeToLazilyUpdateMachineCountThreshold, value => configuration.SafeToLazilyUpdateMachineCountThreshold = value); configuration.EnableReconciliation = !_distributedSettings.Unsafe_DisableReconciliation; configuration.ReconciliationCycleFrequency = TimeSpan.FromMinutes(_distributedSettings.ReconciliationCycleFrequencyMinutes); configuration.ReconciliationMaxCycleSize = _distributedSettings.ReconciliationMaxCycleSize; configuration.ReconciliationMaxRemoveHashesCycleSize = _distributedSettings.ReconciliationMaxRemoveHashesCycleSize; configuration.ReconciliationMaxRemoveHashesAddPercentage = _distributedSettings.ReconciliationMaxRemoveHashesAddPercentage; ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value); ApplyIfNotNull(_distributedSettings.IncrementalCheckpointDegreeOfParallelism, value => configuration.Checkpoint.IncrementalCheckpointDegreeOfParallelism = value); ApplyIfNotNull(_distributedSettings.UseRedisPreventThreadTheftFeature, value => configuration.UsePreventThreadTheftFeature = value); configuration.RedisGlobalStoreConnectionString = ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName)).Secret; if (_distributedSettings.SecondaryGlobalRedisSecretName != null) { configuration.RedisGlobalStoreSecondaryConnectionString = ((PlainTextSecret)GetRequiredSecret( secrets, _distributedSettings.SecondaryGlobalRedisSecretName)).Secret; } ApplyIfNotNull(_distributedSettings.RedisInternalLogSeverity, value => { if (!Enum.TryParse <Severity>(value, out var parsedValue)) { throw new ArgumentException($"Failed to parse `{nameof(_distributedSettings.RedisInternalLogSeverity)}` setting with value `{value}` into type `{nameof(Severity)}`"); } configuration.RedisInternalLogSeverity = parsedValue; }); ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value)); ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => configuration.Checkpoint.RestoreCheckpointAgeThreshold = TimeSpan.FromMinutes(v)); // Need to disable cleaning database on initialization when restore checkpoint age is set. ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => dbConfig.CleanOnInitialize = false); var errorBuilder = new StringBuilder(); var storageCredentials = GetStorageCredentials(secrets, errorBuilder); Contract.Assert(storageCredentials != null && storageCredentials.Length > 0); var blobStoreConfiguration = new BlobCentralStoreConfiguration( credentials: storageCredentials, containerName: _arguments.HostInfo.AppendRingSpecifierIfNeeded("checkpoints", _distributedSettings.UseRingIsolation), checkpointsKey: "checkpoints-eventhub"); ApplyIfNotNull( _distributedSettings.CentralStorageOperationTimeoutInMinutes, value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value)); configuration.CentralStore = blobStoreConfiguration; if (_distributedSettings.UseDistributedCentralStorage) { var distributedCentralStoreConfiguration = new DistributedCentralStoreConfiguration(localCacheRoot) { MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb, PropagationDelay = TimeSpan.FromSeconds(_distributedSettings.CentralStoragePropagationDelaySeconds), PropagationIterations = _distributedSettings.CentralStoragePropagationIterations, MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies }; if (_distributedSettings.UseSelfCheckSettingsForDistributedCentralStorage) { distributedCentralStoreConfiguration.SelfCheckSettings = CreateSelfCheckSettings(_distributedSettings); } distributedCentralStoreConfiguration.TraceFileSystemContentStoreDiagnosticMessages = _distributedSettings.TraceFileSystemContentStoreDiagnosticMessages; ApplyIfNotNull(_distributedSettings.DistributedCentralStoragePeerToPeerCopyTimeoutSeconds, v => distributedCentralStoreConfiguration.PeerToPeerCopyTimeout = TimeSpan.FromSeconds(v)); ApplyIfNotNull(_distributedSettings.DistributedCentralStorageImmutabilityOptimizations, v => distributedCentralStoreConfiguration.ImmutabilityOptimizations = v); configuration.DistributedCentralStore = distributedCentralStoreConfiguration; } var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration( eventHubName: _distributedSettings.EventHubName, eventHubConnectionString: ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName)).Secret, consumerGroupName: _distributedSettings.EventHubConsumerGroupName, epoch: _keySpace + _distributedSettings.EventHubEpoch); dbConfig.Epoch = eventStoreConfiguration.Epoch; configuration.EventStore = eventStoreConfiguration; ApplyIfNotNull( _distributedSettings.MaxEventProcessingConcurrency, value => eventStoreConfiguration.MaxEventProcessingConcurrency = value); ApplyIfNotNull( _distributedSettings.EventBatchSize, value => eventStoreConfiguration.EventBatchSize = value); ApplyIfNotNull( _distributedSettings.EventProcessingMaxQueueSize, value => eventStoreConfiguration.EventProcessingMaxQueueSize = value); }
private async Task ApplySecretSettingsForLlsAsync(RedisContentLocationStoreConfiguration configuration, AbsolutePath localCacheRoot, RocksDbContentLocationDatabaseConfiguration dbConfig) { (var secrets, var errors) = await _secretRetriever.TryRetrieveSecretsAsync(); if (secrets == null) { _logger.Error($"Unable to configure Local Location Store. {errors}"); return; } configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot); if (_distributedSettings.IsMasterEligible) { // Use master selection by setting role to null configuration.Checkpoint.Role = null; } else { // Not master eligible. Set role to worker. configuration.Checkpoint.Role = Role.Worker; } var checkpointConfiguration = configuration.Checkpoint; ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value); ApplyIfNotNull( _distributedSettings.HeartbeatIntervalMinutes, value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.CreateCheckpointIntervalMinutes, value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.RestoreCheckpointIntervalMinutes, value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.SafeToLazilyUpdateMachineCountThreshold, value => configuration.SafeToLazilyUpdateMachineCountThreshold = value); configuration.EnableReconciliation = !_distributedSettings.Unsafe_DisableReconciliation; configuration.ReconciliationCycleFrequency = TimeSpan.FromMinutes(_distributedSettings.ReconciliationCycleFrequencyMinutes); configuration.ReconciliationMaxCycleSize = _distributedSettings.ReconciliationMaxCycleSize; ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value); ApplyIfNotNull(_distributedSettings.IncrementalCheckpointDegreeOfParallelism, value => configuration.Checkpoint.IncrementalCheckpointDegreeOfParallelism = value); configuration.RedisGlobalStoreConnectionString = ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName)).Secret; if (_distributedSettings.SecondaryGlobalRedisSecretName != null) { configuration.RedisGlobalStoreSecondaryConnectionString = ((PlainTextSecret)GetRequiredSecret( secrets, _distributedSettings.SecondaryGlobalRedisSecretName)).Secret; } ApplyIfNotNull( _distributedSettings.ContentLocationReadMode, value => configuration.ReadMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value)); ApplyIfNotNull( _distributedSettings.ContentLocationWriteMode, value => configuration.WriteMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value)); ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value)); ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => configuration.Checkpoint.RestoreCheckpointAgeThreshold = TimeSpan.FromMinutes(v)); // Need to disable cleaning database on initialization when restore checkpoint age is set. ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => dbConfig.CleanOnInitialize = false); var errorBuilder = new StringBuilder(); var storageCredentials = GetStorageCredentials(secrets, errorBuilder); Contract.Assert(storageCredentials != null && storageCredentials.Length > 0); var blobStoreConfiguration = new BlobCentralStoreConfiguration( credentials: storageCredentials, containerName: "checkpoints", checkpointsKey: "checkpoints-eventhub"); ApplyIfNotNull( _distributedSettings.CentralStorageOperationTimeoutInMinutes, value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value)); configuration.CentralStore = blobStoreConfiguration; if (_distributedSettings.UseDistributedCentralStorage) { var distributedCentralStoreConfiguration = new DistributedCentralStoreConfiguration(localCacheRoot) { MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb, PropagationDelay = TimeSpan.FromSeconds( _distributedSettings.CentralStoragePropagationDelaySeconds), PropagationIterations = _distributedSettings.CentralStoragePropagationIterations, MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies }; ApplyIfNotNull(_distributedSettings.DistributedCentralStoragePeerToPeerCopyTimeoutSeconds, v => distributedCentralStoreConfiguration.PeerToPeerCopyTimeout = TimeSpan.FromSeconds(v)); configuration.DistributedCentralStore = distributedCentralStoreConfiguration; } var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration( eventHubName: "eventhub", eventHubConnectionString: ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName)).Secret, consumerGroupName: "$Default", epoch: _keySpace + _distributedSettings.EventHubEpoch); configuration.EventStore = eventStoreConfiguration; ApplyIfNotNull( _distributedSettings.MaxEventProcessingConcurrency, value => eventStoreConfiguration.MaxEventProcessingConcurrency = value); ApplyIfNotNull( _distributedSettings.EventBatchSize, value => eventStoreConfiguration.EventBatchSize = value); ApplyIfNotNull( _distributedSettings.EventProcessingMaxQueueSize, value => eventStoreConfiguration.EventProcessingMaxQueueSize = value); }