private async Task CleanUpRabbitInfrastructureDeploy(NextPipeTask task)
{
var builder = new StringBuilder();
var metadata = task.Metadata as InfrastructureInstallMetadata;
// The task is hanging --> if the number of restarts is 1 then cleanup the rabbitInfrastructure task
// if metadata is null then also cleanup, we dont have required data to do a restart
if (task.Restarts >= 1 || metadata == null)
{
LogHandler.WriteLineVerbose($"Task has already been restarted {task.Restarts} time(s) or metadata is null");
// Task already restarted once, stop task and cleanup infrastructure
builder.AppendLine("Suspending task... Restart limit 1/1 reached. See logs for failure reason and check if manuel cleanup is required");
await _tasksRepository.AppendLog(task.TaskId, task.Logs + builder.ToString());
await _tasksRepository.UpdateStatus(task.TaskId, TaskStatus.Failed, QueueStatus.Suspended);
// Schedule an uninstallInfrastructure cleanup
var id = new Id();
await _tasksRepository.Insert(new NextPipeTask
{
Id = new Id().Value,
TaskId = id.Value,
TaskStatus = TaskStatus.Ready,
QueueStatus = QueueStatus.Pending,
TaskType = TaskType.RabbitInfrastructureUninstall,
TaskPriority = TaskPriority.Fatal,
ReferenceId = task.TaskId,
Hostname = new Hostname().Value,
});
await _eventPublisher.PublishAsync(new UninstallInfrastructureTaskRequestEvent(id));
return;
}
var host = new Hostname();
LogHandler.WriteLineVerbose($"Task was restarted due to previous host death. Restart 1/1 - Attaching task to host: {host.Value}");
builder.AppendLine($"Task was restarted due to previous host death. Restart 1/1 - Attaching task to host: {host.Value}");
// Increment restarts and log
await _tasksRepository.IncrementRestarts(task.TaskId, host.Value, task.Logs + builder.ToString());
LogHandler.WriteLineVerbose("Task was updated with new host.Value and logs");
// Republish infrastructure initialize event
await _eventPublisher.PublishAsync(new InitializeInfrastructureTaskRequestEvent(
new Id(task.TaskId),
new LowerBoundaryReadyReplicas(metadata.LowerBoundaryReadyReplicas),
new ReplicaFailureThreshold(metadata.ReplicaFailureThreshold),
new ReplicaDelaySeconds(metadata.ReplicaDelaySeconds),
new RabbitNumberOfReplicas(metadata.RabbitNumberOfReplicas)));
}
public async Task HandleAsync(InstallModuleEvent evt, CancellationToken ct)
{
// Fetch module first
var module = await _moduleRepository.GetById(evt.ModuleId.Value);
// Update module status to installing
await _moduleRepository.UpdateModuleStatus(module.Id, ModuleStatus.Installing);
// Create a task to handle this installation
NextPipeTask task = await AttachToTaskOrStartNew(evt.TaskId, new Id(module.Id), TaskType.ModuleInstall);
_moduleManager.SetVerboseLogging(true);
await _moduleManager.DeployModule(new ModuleManagerConfig(
new Id(task.TaskId),
new ModuleReplicas(module.DesiredReplicas),
new ModuleName(module.ModuleName),
new ImageName(module.ImageName),
new LoadBalancerConfig(module.LoadBalancerConfig.NeedLoadBalancer, module.LoadBalancerConfig.Port, module.LoadBalancerConfig.TargetPort),
async(id, logHandler) =>
{
await _moduleRepository.UpdateModuleStatus(module.Id, ModuleStatus.Running);
await _tasksRepository.FinishTask(id.Value, TaskStatus.Success, logHandler.GetLog());
},
async(id, logHandler) =>
{
await _moduleRepository.UpdateModuleStatus(module.Id, ModuleStatus.Failed);
await _tasksRepository.FinishTask(id.Value, TaskStatus.Failed, logHandler.GetLog());
},
async(id, logHandler) => { await _tasksRepository.AppendLog(id.Value, logHandler.GetLog()); }));
}
private async Task UpdateCallback(Id taskId, ILogHandler logHandler)
{
await _tasksRepository.AppendLog(taskId.Value, logHandler.GetLog());
}
