public async Task ExecuteAsync(UpdateMonitoringCommand command)
{
var rawDataQueueCount = await _queueClient.MessageCount(_config.RawDataQueueName);
var ingestedQueueCount = await _queueClient.MessageCount(_config.IngestedQueueName);
var mappedQueueCount = await _queueClient.MessageCount(_config.MappedQueueName);
var reducedQueueCount = await _queueClient.MessageCount(_config.ReducedQueueName);
var finalReducedQueueCount = await _queueClient.MessageCount(_config.FinalReducedQueueName);
var commandExecuterQueueCount = await _queueClient.MessageCount(_config.CommandQueueName);
var remoteCommandExecuterQueueCount = await _queueClient.MessageCount(_config.RemoteCommandQueueName);
var workerRecords = await _workerRecordStoreService.GetAllWorkerRecords();
var ingesterWorkerRecords = workerRecords.Where(x => x.Type == "ingester" && !x.HasTerminated);
var mapperWorkerRecords = workerRecords.Where(x => x.Type == "mapper" && !x.HasTerminated);
var reducerWorkerRecords = workerRecords.Where(x => x.Type == "reducer" && !x.HasTerminated);
var finalReducerWorkerRecords = workerRecords.Where(x => x.Type == "finalReducer" && !x.HasTerminated);
var commandExecuterWorkerRecords = workerRecords.Where(x => x.Type == "commandExecuter" && !x.HasTerminated);
var runningIngestersCount = ingesterWorkerRecords.Count();
var runningMappersCount = mapperWorkerRecords.Count();
var runningReducersCount = reducerWorkerRecords.Count();
var runningFinalReducerCount = finalReducerWorkerRecords.Count();
var runningCommandExecuterCount = commandExecuterWorkerRecords.Count();
var htmlTemplate = LoadHtmlTemplate();
var html = htmlTemplate
.Replace("#rawDataQueueCount#", rawDataQueueCount.ToString())
.Replace("#ingestedQueueCount#", ingestedQueueCount.ToString())
.Replace("#mappedQueueCount#", mappedQueueCount.ToString())
.Replace("#reducedQueueCount#", reducedQueueCount.ToString())
.Replace("#runningIngestersCount#", runningIngestersCount.ToString())
.Replace("#runningMappersCount#", runningMappersCount.ToString())
.Replace("#runningReducersCount#", runningReducersCount.ToString())
.Replace("#runningFinalReducersCount#", runningFinalReducerCount.ToString())
.Replace("#finalReducedQueueCount#", finalReducedQueueCount.ToString())
.Replace("#runningCommandExecuterCount#", runningCommandExecuterCount.ToString())
.Replace("#commandExecuterQueueCount#", commandExecuterQueueCount.ToString())
.Replace("#remoteCommandExecuterQueueCount#", remoteCommandExecuterQueueCount.ToString());
using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(html)))
{
await _commandDispatcher.DispatchAsync(new StoreObjectCommand
{
Key = $"{_config.MonitoringFolder}/index.html",
DataStream = memoryStream
});
}
}
public WorkerRecordStoreServiceMockBuilder WithWorkerRecord(string type, DateTime lastPingTime = default(DateTime))
{
if (lastPingTime == default(DateTime))
{
lastPingTime = DateTime.UtcNow;
}
_workerRecords.Add(new WorkerRecord
{
Id = UniqueIdHelper.GenerateUniqueId(),
Type = type,
HasTerminated = false,
ShouldRun = true,
LastPingTime = lastPingTime
});
_workerRecordStoreService.GetAllWorkerRecords().Returns(_workerRecords);
return(this);
}
/// <summary>
/// Invoked periodically to manage runing worker processes
/// </summary>
/// <returns></returns>
public async Task InvokeAsync()
{
Console.WriteLine("Working manager starting...");
var workerRecords = await _workerRecordStoreService.GetAllWorkerRecords();
if (!await ReadyToRunFinalReducer(workerRecords))
{
await RegulateRunningInstances(_config.RawDataQueueName, workerRecords, "ingester", () => new IngestCommand());
await RegulateRunningInstances(_config.IngestedQueueName, workerRecords, "mapper", () => new MapperCommand());
await RegulateRunningInstances(new[] { _config.MappedQueueName, _config.ReducedQueueName }, workerRecords, "reducer", () => new ReducerCommand());
await RegulateRunningInstances(_config.CommandQueueName, workerRecords, "commandQueue", () => new CommandExecuterCommand());
}
else
{
await _dispatcher.DispatchAsync(new FinalReducerCommand());
}
await _dispatcher.DispatchAsync(new UpdateMonitoringCommand());
Thread.Sleep(_config.SleepBetweenWorkerManagerIterationsInMs);
var jobComplete = await JobComplete(workerRecords);
if (jobComplete)
{
await _dispatcher.DispatchAsync(new TerminateCommand());
}
else
{
await _dispatcher.DispatchAsync(new WorkerManagerCommand());
}
await _dispatcher.DispatchAsync(new UpdateMonitoringCommand());
Console.WriteLine("Worker Manager Completed...");
}