public Task ReconciliationOverRealStorage()
{
var checkpointsKey = Guid.NewGuid().ToString();
// Copy and paste a real connection string here.
var storageConnectionString = string.Empty;
// Consider updating this directory if you want to keep data between invocations.
var workingDirectory = TestRootDirectoryPath;
var configuration = new LocalDiskCentralStoreConfiguration(
workingDirectory,
checkpointsKey);
var blobStoreConfiguration = new BlobCentralStoreConfiguration(
credentials: new AzureBlobStorageCredentials(storageConnectionString),
containerName: "checkpoints",
checkpointsKey: checkpointsKey);
var producerMachineLocation = new MachineLocation();
ConfigureWithOneMaster(s =>
{
s.ReconcileMode = ReconciliationMode.Once.ToString();
s.AzureStorageSecretName = Host.StoreSecret("StorageName", storageConnectionString);
});
return(RunTestAsync(
new Context(Logger),
2,
async context =>
{
var master = context.GetMaster();
var worker = context.GetFirstWorker();
var workerId = worker.LocalLocationStore.ClusterState.PrimaryMachineId;
var workerSession = context.Sessions[context.GetFirstWorkerIndex()];
var checkpointState = new CheckpointState(
Role.Worker,
EventSequencePoint.Parse("24382354"),
"MD5:8C4856EA13F6AD59B65D8F6781D2A2F9||DCS||incrementalCheckpoints/24382354.10a0ca0f-d63f-4992-a088-f67bd00abd8a.checkpointInfo.txt|Incremental",
DateTime.Now,
producerMachineLocation,
producerMachineLocation);
// Next heartbeat workers to restore checkpoint
await worker.LocalLocationStore.ProcessStateAsync(new OperationContext(context), checkpointState, inline: true, forceRestore: true).ShouldBeSuccess();
var reconcileResult = await worker.ReconcileAsync(context).ShouldBeSuccess();
Output.WriteLine($"Reconcile result: {reconcileResult}");
}));
}
private async Task ApplySecretSettingsForLlsAsync(RedisContentLocationStoreConfiguration configuration, AbsolutePath localCacheRoot)
{
(var secrets, var errors) = await _secretRetriever.TryRetrieveSecretsAsync();
if (secrets == null)
{
_logger.Error($"Unable to configure Local Location Store. {errors}");
return;
}
configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot);
if (_distributedSettings.IsMasterEligible)
{
// Use master selection by setting role to null
configuration.Checkpoint.Role = null;
}
else
{
// Not master eligible. Set role to worker.
configuration.Checkpoint.Role = Role.Worker;
}
var checkpointConfiguration = configuration.Checkpoint;
ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value);
ApplyIfNotNull(
_distributedSettings.HeartbeatIntervalMinutes,
value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.CreateCheckpointIntervalMinutes,
value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.RestoreCheckpointIntervalMinutes,
value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.SafeToLazilyUpdateMachineCountThreshold,
value => configuration.SafeToLazilyUpdateMachineCountThreshold = value);
configuration.EnableReconciliation = !_distributedSettings.Unsafe_DisableReconciliation;
configuration.ReconciliationCycleFrequency = TimeSpan.FromMinutes(_distributedSettings.ReconciliationCycleFrequencyMinutes);
configuration.ReconciliationMaxCycleSize = _distributedSettings.ReconciliationMaxCycleSize;
ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value);
ApplyIfNotNull(_distributedSettings.IncrementalCheckpointDegreeOfParallelism, value => configuration.Checkpoint.IncrementalCheckpointDegreeOfParallelism = value);
configuration.RedisGlobalStoreConnectionString = ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName)).Secret;
if (_distributedSettings.SecondaryGlobalRedisSecretName != null)
{
configuration.RedisGlobalStoreSecondaryConnectionString = ((PlainTextSecret)GetRequiredSecret(
secrets,
_distributedSettings.SecondaryGlobalRedisSecretName)).Secret;
}
ApplyIfNotNull(
_distributedSettings.ContentLocationReadMode,
value => configuration.ReadMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value));
ApplyIfNotNull(
_distributedSettings.ContentLocationWriteMode,
value => configuration.WriteMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value));
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value));
var errorBuilder = new StringBuilder();
var storageCredentials = GetStorageCredentials(secrets, errorBuilder);
Contract.Assert(storageCredentials != null && storageCredentials.Length > 0);
var blobStoreConfiguration = new BlobCentralStoreConfiguration(
credentials: storageCredentials,
containerName: "checkpoints",
checkpointsKey: "checkpoints-eventhub");
ApplyIfNotNull(
_distributedSettings.CentralStorageOperationTimeoutInMinutes,
value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value));
configuration.CentralStore = blobStoreConfiguration;
if (_distributedSettings.UseDistributedCentralStorage)
{
configuration.DistributedCentralStore = new DistributedCentralStoreConfiguration(localCacheRoot)
{
MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb,
PropagationDelay = TimeSpan.FromSeconds(
_distributedSettings.CentralStoragePropagationDelaySeconds),
PropagationIterations = _distributedSettings.CentralStoragePropagationIterations,
MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies
};
}
var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration(
eventHubName: "eventhub",
eventHubConnectionString: ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName)).Secret,
consumerGroupName: "$Default",
epoch: _keySpace + _distributedSettings.EventHubEpoch);
configuration.EventStore = eventStoreConfiguration;
ApplyIfNotNull(
_distributedSettings.MaxEventProcessingConcurrency,
value => eventStoreConfiguration.MaxEventProcessingConcurrency = value);
ApplyIfNotNull(
_distributedSettings.EventBatchSize,
value => eventStoreConfiguration.EventBatchSize = value);
ApplyIfNotNull(
_distributedSettings.EventProcessingMaxQueueSize,
value => eventStoreConfiguration.EventProcessingMaxQueueSize = value);
}
private async Task ApplySecretSettingsForLlsAsync(
RedisContentLocationStoreConfiguration configuration,
AbsolutePath localCacheRoot,
RocksDbContentLocationDatabaseConfiguration dbConfig)
{
(var secrets, var errors) = await _secretRetriever.TryRetrieveSecretsAsync();
if (secrets == null)
{
_logger.Error($"Unable to configure Local Location Store. {errors}");
return;
}
configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot);
if (_distributedSettings.IsMasterEligible)
{
// Use master selection by setting role to null
configuration.Checkpoint.Role = null;
}
else
{
// Not master eligible. Set role to worker.
configuration.Checkpoint.Role = Role.Worker;
}
// It is important to set the current role of the service, to have non-null Role column
// in all the tracing messages emitted to Kusto.
GlobalInfoStorage.SetServiceRole(configuration.Checkpoint.Role?.ToString() ?? "MasterEligible");
var checkpointConfiguration = configuration.Checkpoint;
ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value);
ApplyIfNotNull(
_distributedSettings.HeartbeatIntervalMinutes,
value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.CreateCheckpointIntervalMinutes,
value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.RestoreCheckpointIntervalMinutes,
value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.RestoreCheckpointTimeoutMinutes,
value => checkpointConfiguration.RestoreCheckpointTimeout = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.UpdateClusterStateIntervalSeconds,
value => checkpointConfiguration.UpdateClusterStateInterval = TimeSpan.FromSeconds(value));
ApplyIfNotNull(_distributedSettings.PacemakerEnabled, v => checkpointConfiguration.PacemakerEnabled = v);
ApplyIfNotNull(_distributedSettings.PacemakerNumberOfBuckets, v => checkpointConfiguration.PacemakerNumberOfBuckets = v);
ApplyIfNotNull(_distributedSettings.PacemakerUseRandomIdentifier, v => checkpointConfiguration.PacemakerUseRandomIdentifier = v);
ApplyIfNotNull(
_distributedSettings.SafeToLazilyUpdateMachineCountThreshold,
value => configuration.SafeToLazilyUpdateMachineCountThreshold = value);
configuration.EnableReconciliation = !_distributedSettings.Unsafe_DisableReconciliation;
configuration.ReconciliationCycleFrequency = TimeSpan.FromMinutes(_distributedSettings.ReconciliationCycleFrequencyMinutes);
configuration.ReconciliationMaxCycleSize = _distributedSettings.ReconciliationMaxCycleSize;
configuration.ReconciliationMaxRemoveHashesCycleSize = _distributedSettings.ReconciliationMaxRemoveHashesCycleSize;
configuration.ReconciliationMaxRemoveHashesAddPercentage = _distributedSettings.ReconciliationMaxRemoveHashesAddPercentage;
ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value);
ApplyIfNotNull(_distributedSettings.IncrementalCheckpointDegreeOfParallelism, value => configuration.Checkpoint.IncrementalCheckpointDegreeOfParallelism = value);
ApplyIfNotNull(_distributedSettings.UseRedisPreventThreadTheftFeature, value => configuration.UsePreventThreadTheftFeature = value);
configuration.RedisGlobalStoreConnectionString = ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName)).Secret;
if (_distributedSettings.SecondaryGlobalRedisSecretName != null)
{
configuration.RedisGlobalStoreSecondaryConnectionString = ((PlainTextSecret)GetRequiredSecret(
secrets,
_distributedSettings.SecondaryGlobalRedisSecretName)).Secret;
}
ApplyIfNotNull(_distributedSettings.RedisInternalLogSeverity, value =>
{
if (!Enum.TryParse <Severity>(value, out var parsedValue))
{
throw new ArgumentException($"Failed to parse `{nameof(_distributedSettings.RedisInternalLogSeverity)}` setting with value `{value}` into type `{nameof(Severity)}`");
}
configuration.RedisInternalLogSeverity = parsedValue;
});
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value));
ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => configuration.Checkpoint.RestoreCheckpointAgeThreshold = TimeSpan.FromMinutes(v));
// Need to disable cleaning database on initialization when restore checkpoint age is set.
ApplyIfNotNull(_distributedSettings.RestoreCheckpointAgeThresholdMinutes, v => dbConfig.CleanOnInitialize = false);
var errorBuilder = new StringBuilder();
var storageCredentials = GetStorageCredentials(secrets, errorBuilder);
Contract.Assert(storageCredentials != null && storageCredentials.Length > 0);
var blobStoreConfiguration = new BlobCentralStoreConfiguration(
credentials: storageCredentials,
containerName: _arguments.HostInfo.AppendRingSpecifierIfNeeded("checkpoints", _distributedSettings.UseRingIsolation),
checkpointsKey: "checkpoints-eventhub");
ApplyIfNotNull(
_distributedSettings.CentralStorageOperationTimeoutInMinutes,
value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value));
configuration.CentralStore = blobStoreConfiguration;
if (_distributedSettings.UseDistributedCentralStorage)
{
var distributedCentralStoreConfiguration = new DistributedCentralStoreConfiguration(localCacheRoot)
{
MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb,
PropagationDelay = TimeSpan.FromSeconds(_distributedSettings.CentralStoragePropagationDelaySeconds),
PropagationIterations = _distributedSettings.CentralStoragePropagationIterations,
MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies
};
if (_distributedSettings.UseSelfCheckSettingsForDistributedCentralStorage)
{
distributedCentralStoreConfiguration.SelfCheckSettings = CreateSelfCheckSettings(_distributedSettings);
}
distributedCentralStoreConfiguration.TraceFileSystemContentStoreDiagnosticMessages = _distributedSettings.TraceFileSystemContentStoreDiagnosticMessages;
ApplyIfNotNull(_distributedSettings.DistributedCentralStoragePeerToPeerCopyTimeoutSeconds, v => distributedCentralStoreConfiguration.PeerToPeerCopyTimeout = TimeSpan.FromSeconds(v));
ApplyIfNotNull(_distributedSettings.DistributedCentralStorageImmutabilityOptimizations, v => distributedCentralStoreConfiguration.ImmutabilityOptimizations = v);
configuration.DistributedCentralStore = distributedCentralStoreConfiguration;
}
var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration(
eventHubName: _distributedSettings.EventHubName,
eventHubConnectionString: ((PlainTextSecret)GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName)).Secret,
consumerGroupName: _distributedSettings.EventHubConsumerGroupName,
epoch: _keySpace + _distributedSettings.EventHubEpoch);
dbConfig.Epoch = eventStoreConfiguration.Epoch;
configuration.EventStore = eventStoreConfiguration;
ApplyIfNotNull(
_distributedSettings.MaxEventProcessingConcurrency,
value => eventStoreConfiguration.MaxEventProcessingConcurrency = value);
ApplyIfNotNull(
_distributedSettings.EventBatchSize,
value => eventStoreConfiguration.EventBatchSize = value);
ApplyIfNotNull(
_distributedSettings.EventProcessingMaxQueueSize,
value => eventStoreConfiguration.EventProcessingMaxQueueSize = value);
}
private async Task ApplyKeyVaultSettingsForLlsAsync(RedisContentLocationStoreConfiguration configuration, AbsolutePath localCacheRoot)
{
var errorBuilder = new StringBuilder();
var secrets = await TryRetrieveKeyVaultSecretsAsync(CancellationToken.None, errorBuilder);
if (secrets == null)
{
_logger.Error($"Unable to configure Local Location Store. {errorBuilder}");
return;
}
configuration.Checkpoint = new CheckpointConfiguration(localCacheRoot);
if (_distributedSettings.IsMasterEligible)
{
// Use master selection by setting role to null
configuration.Checkpoint.Role = null;
}
else
{
// Not master eligible. Set role to worker.
configuration.Checkpoint.Role = Role.Worker;
}
var checkpointConfiguration = configuration.Checkpoint;
ApplyIfNotNull(_distributedSettings.MirrorClusterState, value => configuration.MirrorClusterState = value);
ApplyIfNotNull(
_distributedSettings.HeartbeatIntervalMinutes,
value => checkpointConfiguration.HeartbeatInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.CreateCheckpointIntervalMinutes,
value => checkpointConfiguration.CreateCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.RestoreCheckpointIntervalMinutes,
value => checkpointConfiguration.RestoreCheckpointInterval = TimeSpan.FromMinutes(value));
ApplyIfNotNull(
_distributedSettings.SafeToLazilyUpdateMachineCountThreshold,
value => configuration.SafeToLazilyUpdateMachineCountThreshold = value);
ApplyIfNotNull(_distributedSettings.IsReconciliationEnabled, value => configuration.EnableReconciliation = value);
ApplyIfNotNull(_distributedSettings.UseIncrementalCheckpointing, value => configuration.Checkpoint.UseIncrementalCheckpointing = value);
configuration.RedisGlobalStoreConnectionString = GetRequiredSecret(secrets, _distributedSettings.GlobalRedisSecretName);
if (_distributedSettings.SecondaryGlobalRedisSecretName != null)
{
configuration.RedisGlobalStoreSecondaryConnectionString = GetRequiredSecret(
secrets,
_distributedSettings.SecondaryGlobalRedisSecretName);
}
ApplyIfNotNull(
_distributedSettings.ContentLocationReadMode,
value => configuration.ReadMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value));
ApplyIfNotNull(
_distributedSettings.ContentLocationWriteMode,
value => configuration.WriteMode = (ContentLocationMode)Enum.Parse(typeof(ContentLocationMode), value));
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, value => configuration.LocationEntryExpiry = TimeSpan.FromMinutes(value));
var storageConnectionStrings = GetStorageConnectionStrings(secrets, errorBuilder);
// We already retrieved storage connection strings, so the result should not be null.
Contract.Assert(storageConnectionStrings != null);
var blobStoreConfiguration = new BlobCentralStoreConfiguration(
connectionStrings: storageConnectionStrings,
containerName: "checkpoints",
checkpointsKey: "checkpoints-eventhub");
ApplyIfNotNull(
_distributedSettings.CentralStorageOperationTimeoutInMinutes,
value => blobStoreConfiguration.OperationTimeout = TimeSpan.FromMinutes(value));
configuration.CentralStore = blobStoreConfiguration;
if (_distributedSettings.UseDistributedCentralStorage)
{
configuration.DistributedCentralStore = new DistributedCentralStoreConfiguration(localCacheRoot)
{
MaxRetentionGb = _distributedSettings.MaxCentralStorageRetentionGb,
PropagationDelay = TimeSpan.FromSeconds(
_distributedSettings.CentralStoragePropagationDelaySeconds),
PropagationIterations = _distributedSettings.CentralStoragePropagationIterations,
MaxSimultaneousCopies = _distributedSettings.CentralStorageMaxSimultaneousCopies
};
}
var eventStoreConfiguration = new EventHubContentLocationEventStoreConfiguration(
eventHubName: "eventhub",
eventHubConnectionString: GetRequiredSecret(secrets, _distributedSettings.EventHubSecretName),
consumerGroupName: "$Default",
epoch: _keySpace + _distributedSettings.EventHubEpoch);
configuration.EventStore = eventStoreConfiguration;
ApplyIfNotNull(
_distributedSettings.MaxEventProcessingConcurrency,
value => eventStoreConfiguration.MaxEventProcessingConcurrency = value);
}