public async Task ExecuteAsync(UpdateMonitoringCommand command) { var rawDataQueueCount = await _queueClient.MessageCount(_config.RawDataQueueName); var ingestedQueueCount = await _queueClient.MessageCount(_config.IngestedQueueName); var mappedQueueCount = await _queueClient.MessageCount(_config.MappedQueueName); var reducedQueueCount = await _queueClient.MessageCount(_config.ReducedQueueName); var finalReducedQueueCount = await _queueClient.MessageCount(_config.FinalReducedQueueName); var commandExecuterQueueCount = await _queueClient.MessageCount(_config.CommandQueueName); var remoteCommandExecuterQueueCount = await _queueClient.MessageCount(_config.RemoteCommandQueueName); var workerRecords = await _workerRecordStoreService.GetAllWorkerRecords(); var ingesterWorkerRecords = workerRecords.Where(x => x.Type == "ingester" && !x.HasTerminated); var mapperWorkerRecords = workerRecords.Where(x => x.Type == "mapper" && !x.HasTerminated); var reducerWorkerRecords = workerRecords.Where(x => x.Type == "reducer" && !x.HasTerminated); var finalReducerWorkerRecords = workerRecords.Where(x => x.Type == "finalReducer" && !x.HasTerminated); var commandExecuterWorkerRecords = workerRecords.Where(x => x.Type == "commandExecuter" && !x.HasTerminated); var runningIngestersCount = ingesterWorkerRecords.Count(); var runningMappersCount = mapperWorkerRecords.Count(); var runningReducersCount = reducerWorkerRecords.Count(); var runningFinalReducerCount = finalReducerWorkerRecords.Count(); var runningCommandExecuterCount = commandExecuterWorkerRecords.Count(); var htmlTemplate = LoadHtmlTemplate(); var html = htmlTemplate .Replace("#rawDataQueueCount#", rawDataQueueCount.ToString()) .Replace("#ingestedQueueCount#", ingestedQueueCount.ToString()) .Replace("#mappedQueueCount#", mappedQueueCount.ToString()) .Replace("#reducedQueueCount#", reducedQueueCount.ToString()) .Replace("#runningIngestersCount#", runningIngestersCount.ToString()) .Replace("#runningMappersCount#", runningMappersCount.ToString()) .Replace("#runningReducersCount#", runningReducersCount.ToString()) .Replace("#runningFinalReducersCount#", runningFinalReducerCount.ToString()) .Replace("#finalReducedQueueCount#", finalReducedQueueCount.ToString()) .Replace("#runningCommandExecuterCount#", runningCommandExecuterCount.ToString()) .Replace("#commandExecuterQueueCount#", commandExecuterQueueCount.ToString()) .Replace("#remoteCommandExecuterQueueCount#", remoteCommandExecuterQueueCount.ToString()); using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(html))) { await _commandDispatcher.DispatchAsync(new StoreObjectCommand { Key = $"{_config.MonitoringFolder}/index.html", DataStream = memoryStream }); } }
public WorkerRecordStoreServiceMockBuilder WithWorkerRecord(string type, DateTime lastPingTime = default(DateTime)) { if (lastPingTime == default(DateTime)) { lastPingTime = DateTime.UtcNow; } _workerRecords.Add(new WorkerRecord { Id = UniqueIdHelper.GenerateUniqueId(), Type = type, HasTerminated = false, ShouldRun = true, LastPingTime = lastPingTime }); _workerRecordStoreService.GetAllWorkerRecords().Returns(_workerRecords); return(this); }
/// <summary> /// Invoked periodically to manage runing worker processes /// </summary> /// <returns></returns> public async Task InvokeAsync() { Console.WriteLine("Working manager starting..."); var workerRecords = await _workerRecordStoreService.GetAllWorkerRecords(); if (!await ReadyToRunFinalReducer(workerRecords)) { await RegulateRunningInstances(_config.RawDataQueueName, workerRecords, "ingester", () => new IngestCommand()); await RegulateRunningInstances(_config.IngestedQueueName, workerRecords, "mapper", () => new MapperCommand()); await RegulateRunningInstances(new[] { _config.MappedQueueName, _config.ReducedQueueName }, workerRecords, "reducer", () => new ReducerCommand()); await RegulateRunningInstances(_config.CommandQueueName, workerRecords, "commandQueue", () => new CommandExecuterCommand()); } else { await _dispatcher.DispatchAsync(new FinalReducerCommand()); } await _dispatcher.DispatchAsync(new UpdateMonitoringCommand()); Thread.Sleep(_config.SleepBetweenWorkerManagerIterationsInMs); var jobComplete = await JobComplete(workerRecords); if (jobComplete) { await _dispatcher.DispatchAsync(new TerminateCommand()); } else { await _dispatcher.DispatchAsync(new WorkerManagerCommand()); } await _dispatcher.DispatchAsync(new UpdateMonitoringCommand()); Console.WriteLine("Worker Manager Completed..."); }