private List<DistributedThread> InitMapThreads(int startWorkerNo, int workersCount)
{
var mapFunc = new Func<string, string, IBluepathCommunicationFramework, IEnumerable<string>>((filePath, mapFileName, bluepath) =>
{
if (!(bluepath.Storage is IExtendedStorage))
{
throw new Exception("Mapper requires IExtendedStorage features.");
}
var storage = new BluepathStorage(bluepath.Storage as IExtendedStorage);
var mapProvider = Loader.Load<IMapProvider>(mapFileName, storage);
var mapper = new Mapper(filePath.ToString(), mapProvider.Map, storage);
var mapResult = mapper.PerformMap();
var keys = mapResult.Select(r => r.Key).Distinct().OrderBy(k => k);
var i = 0;
foreach (var res in mapResult)
{
Log.TraceMessage(Log.Activity.Custom,string.Format("[Map] Storing key '{0}', value '{1}'", res.Key, res.Value));
storage.Store(string.Format(Properties.Settings.Default.MapOutputFileName, res.Key, bluepath.ProcessEid, i++), res.Value);
}
return keys;
});
return this.InitThread(startWorkerNo, workersCount, mapFunc);
}
private void PerformMap()
{
while (this.taskQueue.Count > 0)
{
Tuple<string, string> task;
if (!this.taskQueue.TryDequeue(out task))
{
continue;
}
var mapProvider = Loader.Load<IMapProvider>(task.Item2, this.Storage);
var mapper = new Mapper(task.Item1, mapProvider.Map, this.Storage);
var mapResult = mapper.PerformMap();
foreach (var res in mapResult)
{
this.Storage.Store(
string.Format(Properties.Settings.Default.MapOutputFileName, Base64.Encode(res.Key), this.Id, Guid.NewGuid()),
res.Value);
}
}
}