public Task ReconciliationOverRealStorage() { var checkpointsKey = Guid.NewGuid().ToString(); // Copy and paste a real connection string here. var storageConnectionString = string.Empty; // Consider updating this directory if you want to keep data between invocations. var workingDirectory = TestRootDirectoryPath; var configuration = new LocalDiskCentralStoreConfiguration( workingDirectory, checkpointsKey); var blobStoreConfiguration = new BlobCentralStoreConfiguration( credentials: new AzureBlobStorageCredentials(storageConnectionString), containerName: "checkpoints", checkpointsKey: checkpointsKey); var producerMachineLocation = new MachineLocation(); ConfigureWithOneMaster(s => { s.ReconcileMode = ReconciliationMode.Once.ToString(); s.AzureStorageSecretName = Host.StoreSecret("StorageName", storageConnectionString); }); return(RunTestAsync( new Context(Logger), 2, async context => { var master = context.GetMaster(); var worker = context.GetFirstWorker(); var workerId = worker.LocalLocationStore.ClusterState.PrimaryMachineId; var workerSession = context.Sessions[context.GetFirstWorkerIndex()]; var checkpointState = new CheckpointState( Role.Worker, EventSequencePoint.Parse("24382354"), "MD5:8C4856EA13F6AD59B65D8F6781D2A2F9||DCS||incrementalCheckpoints/24382354.10a0ca0f-d63f-4992-a088-f67bd00abd8a.checkpointInfo.txt|Incremental", DateTime.Now, producerMachineLocation, producerMachineLocation); // Next heartbeat workers to restore checkpoint await worker.LocalLocationStore.ProcessStateAsync(new OperationContext(context), checkpointState, inline: true, forceRestore: true).ShouldBeSuccess(); var reconcileResult = await worker.ReconcileAsync(context).ShouldBeSuccess(); Output.WriteLine($"Reconcile result: {reconcileResult}"); })); }
private async Task ApplySecretSettingsForLlsAsync(RedisContentLocationStoreConfiguration configuration, AbsolutePath localCacheRoot) { (var secrets, var errors) = await _secretRetriever.TryRetrieveSecretsAsync(); if (secrets == null) { _logger.Error($"Unable to configure Local Location Store. {errors}"); return; } configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot); if (_distributedSettings.IsMasterEligible) { // Use master selection by setting role to null configuration.Checkpoint.Role = null; } else { // Not master eligible. Set role to worker. configuration.Checkpoint.Role = Role.Worker; } var checkpointConfiguration = configuration.Checkpoint; ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value); ApplyIfNotNull( _distributedSettings.HeartbeatIntervalMinutes, value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.CreateCheckpointIntervalMinutes, value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.RestoreCheckpointIntervalMinutes, value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.SafeToLazilyUpdateMachineCountThreshold, value => configuration.SafeToLazilyUpdateMachineCountThreshold = value); configuration.EnableReconciliation = !_distributedSettings.Unsafe_DisableReconciliation; configuration.ReconciliationCycleFrequency = TimeSpan.FromMinutes(_distributedSettings.ReconciliationCycleFrequencyMinutes); configuration.ReconciliationMaxCycleSize = _distributedSettings.ReconciliationMaxCycleSize; ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value); ApplyIfNotNull(_distributedSettings.IncrementalCheckpointDegreeOfParallelism, value => configuration.Checkpoint.IncrementalCheckpointDegreeOfParallelism = value); configuration.RedisGlobalStoreConnectionString = ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName)).Secret; if (_distributedSettings.SecondaryGlobalRedisSecretName != null) { configuration.RedisGlobalStoreSecondaryConnectionString = ((PlainTextSecret)GetRequiredSecret( secrets, _distributedSettings.SecondaryGlobalRedisSecretName)).Secret; } ApplyIfNotNull( _distributedSettings.ContentLocationReadMode, value => configuration.ReadMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value)); ApplyIfNotNull( _distributedSettings.ContentLocationWriteMode, value => configuration.WriteMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value)); ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value)); var errorBuilder = new StringBuilder(); var storageCredentials = GetStorageCredentials(secrets, errorBuilder); Contract.Assert(storageCredentials != null && storageCredentials.Length > 0); var blobStoreConfiguration = new BlobCentralStoreConfiguration( credentials: storageCredentials, containerName: "checkpoints", checkpointsKey: "checkpoints-eventhub"); ApplyIfNotNull( _distributedSettings.CentralStorageOperationTimeoutInMinutes, value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value)); configuration.CentralStore = blobStoreConfiguration; if (_distributedSettings.UseDistributedCentralStorage) { configuration.DistributedCentralStore = new DistributedCentralStoreConfiguration(localCacheRoot) { MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb, PropagationDelay = TimeSpan.FromSeconds( _distributedSettings.CentralStoragePropagationDelaySeconds), PropagationIterations = _distributedSettings.CentralStoragePropagationIterations, MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies }; } var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration( eventHubName: "eventhub", eventHubConnectionString: ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName)).Secret, consumerGroupName: "$Default", epoch: _keySpace + _distributedSettings.EventHubEpoch); configuration.EventStore = eventStoreConfiguration; ApplyIfNotNull( _distributedSettings.MaxEventProcessingConcurrency, value => eventStoreConfiguration.MaxEventProcessingConcurrency = value); ApplyIfNotNull( _distributedSettings.EventBatchSize, value => eventStoreConfiguration.EventBatchSize = value); ApplyIfNotNull( _distributedSettings.EventProcessingMaxQueueSize, value => eventStoreConfiguration.EventProcessingMaxQueueSize = value); }
private async Task ApplySecretSettingsForLlsAsync( RedisContentLocationStoreConfiguration configuration, AbsolutePath localCacheRoot, RocksDbContentLocationDatabaseConfiguration dbConfig) { (var secrets, var errors) = await _secretRetriever.TryRetrieveSecretsAsync(); if (secrets == null) { _logger.Error($"Unable to configure Local Location Store. {errors}"); return; } configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot); if (_distributedSettings.IsMasterEligible) { // Use master selection by setting role to null configuration.Checkpoint.Role = null; } else { // Not master eligible. Set role to worker. configuration.Checkpoint.Role = Role.Worker; } // It is important to set the current role of the service, to have non-null Role column // in all the tracing messages emitted to Kusto. GlobalInfoStorage.SetServiceRole(configuration.Checkpoint.Role?.ToString() ?? "MasterEligible"); var checkpointConfiguration = configuration.Checkpoint; ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value); ApplyIfNotNull( _distributedSettings.HeartbeatIntervalMinutes, value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.CreateCheckpointIntervalMinutes, value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.RestoreCheckpointIntervalMinutes, value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.RestoreCheckpointTimeoutMinutes, value => checkpointConfiguration.RestoreCheckpointTimeout = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.UpdateClusterStateIntervalSeconds, value => checkpointConfiguration.UpdateClusterStateInterval = TimeSpan.FromSeconds(value)); ApplyIfNotNull(_distributedSettings.PacemakerEnabled, v => checkpointConfiguration.PacemakerEnabled = v); ApplyIfNotNull(_distributedSettings.PacemakerNumberOfBuckets, v => checkpointConfiguration.PacemakerNumberOfBuckets = v); ApplyIfNotNull(_distributedSettings.PacemakerUseRandomIdentifier, v => checkpointConfiguration.PacemakerUseRandomIdentifier = v); ApplyIfNotNull( _distributedSettings.SafeToLazilyUpdateMachineCountThreshold, value => configuration.SafeToLazilyUpdateMachineCountThreshold = value); configuration.EnableReconciliation = !_distributedSettings.Unsafe_DisableReconciliation; configuration.ReconciliationCycleFrequency = TimeSpan.FromMinutes(_distributedSettings.ReconciliationCycleFrequencyMinutes); configuration.ReconciliationMaxCycleSize = _distributedSettings.ReconciliationMaxCycleSize; configuration.ReconciliationMaxRemoveHashesCycleSize = _distributedSettings.ReconciliationMaxRemoveHashesCycleSize; configuration.ReconciliationMaxRemoveHashesAddPercentage = _distributedSettings.ReconciliationMaxRemoveHashesAddPercentage; ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value); ApplyIfNotNull(_distributedSettings.IncrementalCheckpointDegreeOfParallelism, value => configuration.Checkpoint.IncrementalCheckpointDegreeOfParallelism = value); ApplyIfNotNull(_distributedSettings.UseRedisPreventThreadTheftFeature, value => configuration.UsePreventThreadTheftFeature = value); configuration.RedisGlobalStoreConnectionString = ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName)).Secret; if (_distributedSettings.SecondaryGlobalRedisSecretName != null) { configuration.RedisGlobalStoreSecondaryConnectionString = ((PlainTextSecret)GetRequiredSecret( secrets, _distributedSettings.SecondaryGlobalRedisSecretName)).Secret; } ApplyIfNotNull(_distributedSettings.RedisInternalLogSeverity, value => { if (!Enum.TryParse <Severity>(value, out var parsedValue)) { throw new ArgumentException($"Failed to parse `{nameof(_distributedSettings.RedisInternalLogSeverity)}` setting with value `{value}` into type `{nameof(Severity)}`"); } configuration.RedisInternalLogSeverity = parsedValue; }); ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value)); ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => configuration.Checkpoint.RestoreCheckpointAgeThreshold = TimeSpan.FromMinutes(v)); // Need to disable cleaning database on initialization when restore checkpoint age is set. ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => dbConfig.CleanOnInitialize = false); var errorBuilder = new StringBuilder(); var storageCredentials = GetStorageCredentials(secrets, errorBuilder); Contract.Assert(storageCredentials != null && storageCredentials.Length > 0); var blobStoreConfiguration = new BlobCentralStoreConfiguration( credentials: storageCredentials, containerName: _arguments.HostInfo.AppendRingSpecifierIfNeeded("checkpoints", _distributedSettings.UseRingIsolation), checkpointsKey: "checkpoints-eventhub"); ApplyIfNotNull( _distributedSettings.CentralStorageOperationTimeoutInMinutes, value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value)); configuration.CentralStore = blobStoreConfiguration; if (_distributedSettings.UseDistributedCentralStorage) { var distributedCentralStoreConfiguration = new DistributedCentralStoreConfiguration(localCacheRoot) { MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb, PropagationDelay = TimeSpan.FromSeconds(_distributedSettings.CentralStoragePropagationDelaySeconds), PropagationIterations = _distributedSettings.CentralStoragePropagationIterations, MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies }; if (_distributedSettings.UseSelfCheckSettingsForDistributedCentralStorage) { distributedCentralStoreConfiguration.SelfCheckSettings = CreateSelfCheckSettings(_distributedSettings); } distributedCentralStoreConfiguration.TraceFileSystemContentStoreDiagnosticMessages = _distributedSettings.TraceFileSystemContentStoreDiagnosticMessages; ApplyIfNotNull(_distributedSettings.DistributedCentralStoragePeerToPeerCopyTimeoutSeconds, v => distributedCentralStoreConfiguration.PeerToPeerCopyTimeout = TimeSpan.FromSeconds(v)); ApplyIfNotNull(_distributedSettings.DistributedCentralStorageImmutabilityOptimizations, v => distributedCentralStoreConfiguration.ImmutabilityOptimizations = v); configuration.DistributedCentralStore = distributedCentralStoreConfiguration; } var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration( eventHubName: _distributedSettings.EventHubName, eventHubConnectionString: ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName)).Secret, consumerGroupName: _distributedSettings.EventHubConsumerGroupName, epoch: _keySpace + _distributedSettings.EventHubEpoch); dbConfig.Epoch = eventStoreConfiguration.Epoch; configuration.EventStore = eventStoreConfiguration; ApplyIfNotNull( _distributedSettings.MaxEventProcessingConcurrency, value => eventStoreConfiguration.MaxEventProcessingConcurrency = value); ApplyIfNotNull( _distributedSettings.EventBatchSize, value => eventStoreConfiguration.EventBatchSize = value); ApplyIfNotNull( _distributedSettings.EventProcessingMaxQueueSize, value => eventStoreConfiguration.EventProcessingMaxQueueSize = value); }
private async Task ApplyKeyVaultSettingsForLlsAsync(RedisContentLocationStoreConfiguration configuration, AbsolutePath localCacheRoot) { var errorBuilder = new StringBuilder(); var secrets = await TryRetrieveKeyVaultSecretsAsync(CancellationToken.None, errorBuilder); if (secrets == null) { _logger.Error($"Unable to configure Local Location Store. {errorBuilder}"); return; } configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot); if (_distributedSettings.IsMasterEligible) { // Use master selection by setting role to null configuration.Checkpoint.Role = null; } else { // Not master eligible. Set role to worker. configuration.Checkpoint.Role = Role.Worker; } var checkpointConfiguration = configuration.Checkpoint; ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value); ApplyIfNotNull( _distributedSettings.HeartbeatIntervalMinutes, value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.CreateCheckpointIntervalMinutes, value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.RestoreCheckpointIntervalMinutes, value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value)); ApplyIfNotNull( _distributedSettings.SafeToLazilyUpdateMachineCountThreshold, value => configuration.SafeToLazilyUpdateMachineCountThreshold = value); ApplyIfNotNull(_distributedSettings.IsReconciliationEnabled, value => configuration.EnableReconciliation = value); ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value); configuration.RedisGlobalStoreConnectionString = GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName); if (_distributedSettings.SecondaryGlobalRedisSecretName != null) { configuration.RedisGlobalStoreSecondaryConnectionString = GetRequiredSecret( secrets, _distributedSettings.SecondaryGlobalRedisSecretName); } ApplyIfNotNull( _distributedSettings.ContentLocationReadMode, value => configuration.ReadMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value)); ApplyIfNotNull( _distributedSettings.ContentLocationWriteMode, value => configuration.WriteMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value)); ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value)); var storageConnectionStrings = GetStorageConnectionStrings(secrets, errorBuilder); // We already retrieved storage connection strings, so the result should not be null. Contract.Assert(storageConnectionStrings != null); var blobStoreConfiguration = new BlobCentralStoreConfiguration( connectionStrings: storageConnectionStrings, containerName: "checkpoints", checkpointsKey: "checkpoints-eventhub"); ApplyIfNotNull( _distributedSettings.CentralStorageOperationTimeoutInMinutes, value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value)); configuration.CentralStore = blobStoreConfiguration; if (_distributedSettings.UseDistributedCentralStorage) { configuration.DistributedCentralStore = new DistributedCentralStoreConfiguration(localCacheRoot) { MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb, PropagationDelay = TimeSpan.FromSeconds( _distributedSettings.CentralStoragePropagationDelaySeconds), PropagationIterations = _distributedSettings.CentralStoragePropagationIterations, MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies }; } var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration( eventHubName: "eventhub", eventHubConnectionString: GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName), consumerGroupName: "$Default", epoch: _keySpace + _distributedSettings.EventHubEpoch); configuration.EventStore = eventStoreConfiguration; ApplyIfNotNull( _distributedSettings.MaxEventProcessingConcurrency, value => eventStoreConfiguration.MaxEventProcessingConcurrency = value); }