private async Task <Response> InitiateLongRunningProcess(NextPipeProcessType processType, string cmdName, Func <Task> func) { // Check if there exists a Process of respective type which is already running... LogHandler.WriteLineVerbose($"Request processLock for processType: {processType}"); var processLock = await RequestProcessLock(processType); if (processLock == null) { // Request for process lock was not successful return unsuccesfull cmd and try new cleanup in 30 secs LogHandler.WriteLineVerbose($"Couldn't receive processLock for type:{processType}, occupied by other host: {new Hostname().Value} waiting for next {processType} session"); return(Response.Unsuccessful()); } LogHandler.WriteLineVerbose($"ProcessLock of type: {processType} received for cmd: {cmdName}"); try { await func(); } catch (Exception ex) { LogHandler.WriteLineVerbose($"An exception was thrown while executing long running process of type: {processType} for cmd: {cmdName} --> {ex.Message}"); Console.WriteLine(ex); } LogHandler.WriteLineVerbose($"{processType} process done, deleting processLock with id: {processLock.Id}"); // The process is done, remove the processLock await _processLockRepository.Delete(processLock.Id); LogHandler.WriteLineVerbose($"Deleted processLock withId: {processLock.Id} for process {processType}"); return(Response.Success()); }
/// <summary> /// Returns null if the method was not able to assign a processLock. Else returns a processLock /// </summary> /// <param name="processType"></param> /// <returns></returns> private async Task <ProcessLock> RequestProcessLock(NextPipeProcessType processType) { // Find process of processType var process = await _processLockRepository.FindProcessLockByProcessType(processType); if (process != null) { LogHandler.WriteLineVerbose($"Process already running on host: {process.Hostname}"); // The process is already running - Make sure that the processLock is not assigned to a dead host var hostPods = await _kubectlHelper.GetPodsByCustomNameFilter(NEXTPIPE_DEPLOYMENT_NAME, ShellHelper.IdenticalStart); LogHandler.WriteLineVerbose("Running hosts"); foreach (var hostPod in hostPods) { Console.WriteLine($"- {hostPod.Metadata.Name}"); } if (!hostPods.Any(t => t.Metadata.Name.Trim().ToLower().Equals(process.Hostname.Trim().ToLower()))) { LogHandler.WriteLineVerbose( $"Process was hanging on dead host: {process.Hostname}. Rescheduling the process to host: {new Hostname().Value}"); // The hostname of the process does not match any of the current hosts // Re-schedule the CleanupHangingTaskCommand by deleting and inserting a new processLock // Attached to this host return(await _processLockRepository.ReplaceProcessLock(new ProcessLock { Hostname = new Hostname().Value, Id = new Id().Value, ProcessId = new Id().Value, NextPipeProcessType = processType.ToString() }, process)); } } else { LogHandler.WriteLineVerbose($"No process of type: {processType} was running trying to request processLock"); // The process is not running, create a processLock for this host // This might fail if another replica beats us to the finish line return(await _processLockRepository.InsertAndReturn(new ProcessLock { Hostname = new Hostname().Value, Id = new Id().Value, ProcessId = new Id().Value, NextPipeProcessType = processType.ToString() }, new Hostname().Value)); } return(null); }
public async Task <ProcessLock> FindProcessLockByProcessType(NextPipeProcessType type) { return(await Collection().Find(t => t.NextPipeProcessType == type.ToString()).SingleOrDefaultAsync()); }