private async Task Deploy(IRabbitDeploymentManagerConfiguration config, bool abortOnFailure = false)
{
// Check if statefulset is already deployed
await _logHandler.WriteCmd($"{nameof(RabbitDeploymentManager)}.{nameof(Deploy)}", verboseLogging);
await _logHandler.WriteLine("Validating statefulset is running...", verboseLogging);
var rabbitStatefulSetIsRunning = _kubectlHelper.ValidateStatefulsetIsRunning(RABBIT_MQ_STATEFULSET);
if (rabbitStatefulSetIsRunning)
{
await _logHandler.WriteLine($"Rabbit Statefulset is running -->", verboseLogging);
await ValidateRabbitMQDeployment(config);
}
else
{
if (abortOnFailure)
{
await _logHandler.WriteLine("Failed to install rabbitMQ infrastructure", verboseLogging);
await Cleanup();
await _logHandler.WriteLine($"CleanUp Complete --> config.FailureCallback()", verboseLogging);
await config.FailureCallback(config.TaskId, _logHandler);
return;
}
await InstallHelmAndRabbitMQ(config);
// Once helm has installed and rabbitMQ has been provisioned to the cluster by helm retry the init call
// else abort the process...
await Deploy(config, true);
}
}
private async Task InstallHelmAndRabbitMQ(IRabbitDeploymentManagerConfiguration config)
{
await _logHandler.WriteLine("No existing RabbitMQ infrastructure --> Provision RabbitMQ infrastructure", verboseLogging);
await _helmManager.InstallHelm(_logHandler, verboseLogging);
await _helmManager.InstallRabbitMQ(RABBIT_MQ_STATEFULSET, _logHandler, verboseLogging, config.RabbitNumberOfReplicas);
await Task.Delay(30.SecToMillis());
}
private async Task<bool> WaitForLowerBoundaryReplicas(IRabbitDeploymentManagerConfiguration config, string statefulSetname, string nameSpace = "default")
{
// true as long as none of the constraints are met
var failedAttempts = 0;
var readyReplicas = _kubectlHelper.GetNumberOfStatefulsetReadyReplicas(statefulSetname, nameSpace);
await _logHandler.WriteLine($"lowerBoundaryReplicas={config.LowerBoundaryReadyReplicas}, readyReplicas={readyReplicas}. {config.LowerBoundaryReadyReplicas - readyReplicas} ready replica(s) needed for operations. Attempt {failedAttempts}/{config.ReplicaFailureThreshold}",
verboseLogging, LogHandler.InProgressTemplate);
if (readyReplicas >= config.LowerBoundaryReadyReplicas)
{
return true;
}
await _logHandler.WriteLine($"Waiting for ready replicas... {config.ReplicaDelaySeconds}", verboseLogging);
// Wait the initial delay
await Task.Delay(config.ReplicaDelaySeconds.SecToMillis());
while (true)
{
var rReplicas = _kubectlHelper.GetNumberOfStatefulsetReadyReplicas(statefulSetname, nameSpace);
if (rReplicas >= config.LowerBoundaryReadyReplicas)
{
return true;
}
// Increment the failed attempts
failedAttempts++;
if (failedAttempts >= config.ReplicaFailureThreshold)
{
return false;
}
await _logHandler.WriteLine($"lowerBoundaryReplicas={config.LowerBoundaryReadyReplicas}, readyReplicas={readyReplicas}. {config.LowerBoundaryReadyReplicas - readyReplicas} ready replica(s) needed for operations. Attempt {failedAttempts}/{config.ReplicaFailureThreshold}",
verboseLogging, LogHandler.InProgressTemplate);
await Task.Delay(config.ReplicaDelaySeconds.SecToMillis());
}
}
private async Task ValidateRabbitMQDeployment(IRabbitDeploymentManagerConfiguration config)
{
await _logHandler.WriteCmd($"{nameof(RabbitDeploymentManager)}.{nameof(ValidateRabbitMQDeployment)}", verboseLogging);
await _logHandler.WriteLine("Waiting for lowerboundary replicas to come online", verboseLogging);
// Validate that at least lowerBoundaryReplicas are running for availability across the cluster
var isClusterReady = await WaitForLowerBoundaryReplicas(config, RABBIT_MQ_STATEFULSET);
if (isClusterReady)
{
await _logHandler.WriteLine("RabbitMQ cluster is running with desired lowerBoundary replicas online -->", verboseLogging);
await CreateRabbitMQService();
await _logHandler.WriteLine("RabbitMQ successfully installed with desired configuration", verboseLogging);
await _logHandler.WriteCmd($"{nameof(config.SuccessCallback)}()", verboseLogging);
await config.SuccessCallback(config.TaskId, _logHandler);
}
else
{
await _logHandler.WriteLine("RabbitMQ failed to get lowerBoundary replicas running", verboseLogging);
await Cleanup();
await _logHandler.WriteLine($"CleanUp Complete --> config.FailureCallback()", verboseLogging);
await config.FailureCallback(config.TaskId, _logHandler);
}
}
/// <summary>
/// Validate and or provision the rabbitMQ infrastructure.
/// </summary>
/// <param name="lowerBoundaryReplicas"></param>
/// <param name="failureThreshold"></param>
/// <param name="trialsDelaySec"></param>
/// <returns></returns>
public async Task Deploy(IRabbitDeploymentManagerConfiguration config)
{
// Attach taskId and updateCallback to the logHandler
AttachTaskIdAndUpdateHandler(config.TaskId, config.UpdateCallback);
await Deploy(config, false);
}