private async Task CleanUpRabbitInfrastructureDeploy(NextPipeTask task) { var builder = new StringBuilder(); var metadata = task.Metadata as InfrastructureInstallMetadata; // The task is hanging --> if the number of restarts is 1 then cleanup the rabbitInfrastructure task // if metadata is null then also cleanup, we dont have required data to do a restart if (task.Restarts >= 1 || metadata == null) { LogHandler.WriteLineVerbose($"Task has already been restarted {task.Restarts} time(s) or metadata is null"); // Task already restarted once, stop task and cleanup infrastructure builder.AppendLine("Suspending task... Restart limit 1/1 reached. See logs for failure reason and check if manuel cleanup is required"); await _tasksRepository.AppendLog(task.TaskId, task.Logs + builder.ToString()); await _tasksRepository.UpdateStatus(task.TaskId, TaskStatus.Failed, QueueStatus.Suspended); // Schedule an uninstallInfrastructure cleanup var id = new Id(); await _tasksRepository.Insert(new NextPipeTask { Id = new Id().Value, TaskId = id.Value, TaskStatus = TaskStatus.Ready, QueueStatus = QueueStatus.Pending, TaskType = TaskType.RabbitInfrastructureUninstall, TaskPriority = TaskPriority.Fatal, ReferenceId = task.TaskId, Hostname = new Hostname().Value, }); await _eventPublisher.PublishAsync(new UninstallInfrastructureTaskRequestEvent(id)); return; } var host = new Hostname(); LogHandler.WriteLineVerbose($"Task was restarted due to previous host death. Restart 1/1 - Attaching task to host: {host.Value}"); builder.AppendLine($"Task was restarted due to previous host death. Restart 1/1 - Attaching task to host: {host.Value}"); // Increment restarts and log await _tasksRepository.IncrementRestarts(task.TaskId, host.Value, task.Logs + builder.ToString()); LogHandler.WriteLineVerbose("Task was updated with new host.Value and logs"); // Republish infrastructure initialize event await _eventPublisher.PublishAsync(new InitializeInfrastructureTaskRequestEvent( new Id(task.TaskId), new LowerBoundaryReadyReplicas(metadata.LowerBoundaryReadyReplicas), new ReplicaFailureThreshold(metadata.ReplicaFailureThreshold), new ReplicaDelaySeconds(metadata.ReplicaDelaySeconds), new RabbitNumberOfReplicas(metadata.RabbitNumberOfReplicas))); }
public async Task HandleAsync(InstallModuleEvent evt, CancellationToken ct) { // Fetch module first var module = await _moduleRepository.GetById(evt.ModuleId.Value); // Update module status to installing await _moduleRepository.UpdateModuleStatus(module.Id, ModuleStatus.Installing); // Create a task to handle this installation NextPipeTask task = await AttachToTaskOrStartNew(evt.TaskId, new Id(module.Id), TaskType.ModuleInstall); _moduleManager.SetVerboseLogging(true); await _moduleManager.DeployModule(new ModuleManagerConfig( new Id(task.TaskId), new ModuleReplicas(module.DesiredReplicas), new ModuleName(module.ModuleName), new ImageName(module.ImageName), new LoadBalancerConfig(module.LoadBalancerConfig.NeedLoadBalancer, module.LoadBalancerConfig.Port, module.LoadBalancerConfig.TargetPort), async(id, logHandler) => { await _moduleRepository.UpdateModuleStatus(module.Id, ModuleStatus.Running); await _tasksRepository.FinishTask(id.Value, TaskStatus.Success, logHandler.GetLog()); }, async(id, logHandler) => { await _moduleRepository.UpdateModuleStatus(module.Id, ModuleStatus.Failed); await _tasksRepository.FinishTask(id.Value, TaskStatus.Failed, logHandler.GetLog()); }, async(id, logHandler) => { await _tasksRepository.AppendLog(id.Value, logHandler.GetLog()); })); }
private async Task UpdateCallback(Id taskId, ILogHandler logHandler) { await _tasksRepository.AppendLog(taskId.Value, logHandler.GetLog()); }