public async Task InvokeAsync()
{
var instanceWorkerId = _workerRecordStoreService.GenerateUniqueId();
Console.WriteLine($"CommandExecuter {instanceWorkerId} starting...");
await _workerRecordStoreService.RecordPing("commandExecuter", instanceWorkerId);
var commandQueueMessages = await _queueClient.Dequeue(_config.CommandQueueName, _config.MaxQueueItemsBatchSizeToProcessPerWorker);
if (commandQueueMessages.Count == 0)
{
await _workerRecordStoreService.RecordHasTerminated("commandExecuter", instanceWorkerId);
return;
}
foreach (var commandQueueMessage in commandQueueMessages)
{
var serializerSettings = new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.All
};
var command = JsonConvert.DeserializeObject <NoResultCommandWrapper>(commandQueueMessage.Message, serializerSettings);
await _directCommandExecuter.ExecuteAsync(command);
await _queueClient.MessageProcessed(_config.CommandQueueName, commandQueueMessage.MessageId);
}
await _workerRecordStoreService.RecordHasTerminated("commandExecuter", instanceWorkerId);
Console.WriteLine($"Mapper {instanceWorkerId} Terminated");
}
public async Task InvokeAsync()
{
var instanceWorkerId = _workerRecordStoreService.GenerateUniqueId();
await _workerRecordStoreService.RecordPing("ingester", instanceWorkerId);
var rawDataQueueMessages = await _queueClient.Dequeue(_config.RawDataQueueName, _config.MaxQueueItemsBatchSizeToProcessPerWorker);
if (rawDataQueueMessages.Count == 0)
{
await _workerRecordStoreService.RecordHasTerminated("ingester", instanceWorkerId);
return;
}
foreach (var rawDataQueueMessage in rawDataQueueMessages)
{
var rawDataObjectKey = rawDataQueueMessage.Message;
Stream rawDataObjectStream = await _commandDispatcher.DispatchAsync(new RetrieveObjectCommand { Key = rawDataQueueMessage.Message });
var objectName = _keyRegex.Match(rawDataObjectKey).Groups["objectName"].Value;
using (var objectReader = new StreamReader(rawDataObjectStream))
{
while (!objectReader.EndOfStream)
{
using (var memoryStream = new MemoryStream())
{
var ingestedObjectKey = $"{_config.IngestedFolder}/{objectName}-{Guid.NewGuid()}";
using (var sw = new StreamWriter(memoryStream))
{
for (int i = 0; i < _config.IngesterMaxLinesPerFile && !objectReader.EndOfStream; i++)
{
var line = await objectReader.ReadLineAsync();
await sw.WriteLineAsync(line);
}
await sw.FlushAsync();
await _commandDispatcher.DispatchAsync(new StoreObjectCommand
{
Key = ingestedObjectKey,
DataStream = memoryStream
});
}
await _queueClient.Enqueue(_config.IngestedQueueName, ingestedObjectKey);
}
}
}
await _queueClient.MessageProcessed(_config.RawDataQueueName, rawDataQueueMessage.MessageId);
}
await _workerRecordStoreService.RecordHasTerminated("ingester", instanceWorkerId);
}
public async Task InvokeAsync()
{
var instanceWorkerId = _workerRecordStoreService.GenerateUniqueId();
Console.WriteLine($"Mapper {instanceWorkerId} starting...");
await _workerRecordStoreService.RecordPing("mapper", instanceWorkerId);
var ingestedQueueMessages = await _queueClient.Dequeue(_config.IngestedQueueName, _config.MaxQueueItemsBatchSizeToProcessPerWorker);
if (ingestedQueueMessages.Count == 0)
{
await _workerRecordStoreService.RecordHasTerminated("mapper", instanceWorkerId);
return;
}
foreach (var ingestedQueueMessage in ingestedQueueMessages)
{
await _workerRecordStoreService.RecordPing("mapper", instanceWorkerId);
Stream ingestedObjectStream = await _commandDispatcher.DispatchAsync(new RetrieveObjectCommand { Key = ingestedQueueMessage.Message });
using (var streamReader = new StreamReader(ingestedObjectStream))
{
var lines = new List <string>();
while (!streamReader.EndOfStream)
{
var line = await streamReader.ReadLineAsync();
lines.Add(line);
}
await _commandDispatcher.DispatchAsync(new BatchMapDataCommand
{
Lines = lines,
ContextQueueMessage = ingestedQueueMessage
});
}
await _queueClient.MessageProcessed(_config.IngestedQueueName, ingestedQueueMessage.MessageId);
}
await _workerRecordStoreService.RecordHasTerminated("mapper", instanceWorkerId);
Console.WriteLine($"Mapper {instanceWorkerId} Terminated");
}
public async Task InvokeAsync()
{
var instanceWorkerId = _workerRecordStoreService.GenerateUniqueId();
await _workerRecordStoreService.RecordPing("reducer", instanceWorkerId);
var mappedQueueMessages = await _queueClient.Dequeue(_config.MappedQueueName, _config.MaxQueueItemsBatchSizeToProcessPerWorker);
var reducedQueueMessages = await _queueClient.Dequeue(_config.ReducedQueueName, _config.MaxQueueItemsBatchSizeToProcessPerWorker);
if (mappedQueueMessages.Count == 0 && reducedQueueMessages.Count <= 1)
{
foreach (var reducedQueueMessage in reducedQueueMessages)
{
await _queueClient.ReturnMessageToQueue(_config.ReducedQueueName, reducedQueueMessage.MessageId);
}
await _workerRecordStoreService.RecordHasTerminated("reducer", instanceWorkerId);
return;
}
var queueMessages = new List <QueueMessage>();
queueMessages.AddRange(mappedQueueMessages);
queueMessages.AddRange(reducedQueueMessages);
var inputCounts = new List <CompressedMostAccidentProneData>();
foreach (var queueMessage in queueMessages)
{
Stream mappedObjectStream = await _commandDispatcher.DispatchAsync(new RetrieveObjectCommand { Key = queueMessage.Message });
using (var streamReader = new StreamReader(mappedObjectStream))
{
while (!streamReader.EndOfStream)
{
var line = await streamReader.ReadLineAsync();
try
{
var keyValuePairs = JsonConvert.DeserializeObject <List <CompressedMostAccidentProneData> >(line,
new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.Auto
});
inputCounts.AddRange(keyValuePairs);
}
catch (JsonSerializationException e)
{
Console.WriteLine($"Error white deserialising value [{line}]");
}
}
}
}
await _commandDispatcher.DispatchAsync(new BatchReduceDataCommand
{
InputKeyValuePairs2 = inputCounts,
ProcessedMessageIdsHash = ProcessedMessageIdsHash(queueMessages),
QueueMessages = queueMessages.Select(x => x.Message).ToArray()
});
// Without this being in a transaction, there is the risk of incorrect results
MarkProcessed(_config.MappedQueueName, mappedQueueMessages);
MarkProcessed(_config.ReducedQueueName, reducedQueueMessages);
await _workerRecordStoreService.RecordHasTerminated("reducer", instanceWorkerId);
// Transaction end
}
public async Task InvokeAsync()
{
var instanceWorkerId = _workerRecordStoreService.GenerateUniqueId();
await _workerRecordStoreService.RecordPing("finalReducer", instanceWorkerId);
var reducedQueueMessages = await _queueClient.Dequeue(_config.ReducedQueueName, _config.MaxQueueItemsBatchSizeToProcessPerWorker);
if (reducedQueueMessages.Count != 1)
{
await _workerRecordStoreService.RecordHasTerminated("finalReducer", instanceWorkerId);
//TODO consider logging and reutrning without throwing an exception
throw new ApplicationException("Expected to find just one message on the reducer queue");
}
var reducedQueueMessage = reducedQueueMessages.First();
using (var streamReader = new StreamReader(await _commandDispatcher.DispatchAsync(new RetrieveObjectCommand {
Key = reducedQueueMessage.Message
})))
using (var memoryStream = new MemoryStream())
using (var streamWriter = new StreamWriter(memoryStream))
{
if (!streamReader.EndOfStream)
{
var line = await streamReader.ReadLineAsync();
var keyValuePairs = JsonConvert.DeserializeObject <List <CompressedMostAccidentProneData> >(line,
new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.Auto
});
foreach (var keyValuePair in keyValuePairs)
{
var linesToWrite = (await _commandDispatcher.DispatchAsync(
new FinalReducerFuncCommand
{
CompressedMostAccidentProneData = keyValuePair
})).Result;
foreach (var lineToWrite in linesToWrite)
{
await streamWriter.WriteLineAsync(lineToWrite);
}
}
await streamWriter.FlushAsync();
var finalObjectKey = $"{_config.FinalReducedFolder}/{Guid.NewGuid()}";
await _commandDispatcher.DispatchAsync(new StoreObjectCommand
{
Key = finalObjectKey,
DataStream = memoryStream
});
await _queueClient.Enqueue(_config.FinalReducedQueueName, finalObjectKey);
}
}
await _queueClient.MessageProcessed(_config.ReducedQueueName, reducedQueueMessage.MessageId);
await _workerRecordStoreService.RecordHasTerminated("finalReducer", instanceWorkerId);
}