private void PerformReduce()
{
while (this.taskQueue.Count > 0)
{
Tuple<string, string> task;
if (!this.taskQueue.TryDequeue(out task))
{
continue;
}
var reduceProvider = Loader.Load<IReduceProvider>(task.Item2, this.Storage);
var reducer = new Reducer(task.Item1, reduceProvider.Reduce, this.Storage);
var reduceResult = reducer.PerformReduce();
this.Storage.Store(
string.Format(Properties.Settings.Default.ReduceOutputFileName, Base64.Encode(reduceResult.Key), this.Id, Guid.NewGuid()),
reduceResult.Value);
}
}
private List<DistributedThread> InitReduceThreads(int startWorkerNo, int workersCount)
{
var reduceFunc = new Func<string, string, IBluepathCommunicationFramework, IEnumerable<string>>((filePath, reduceFuncName, bluepath) =>
{
if (!(bluepath.Storage is IExtendedStorage))
{
throw new Exception("Reducer requires IExtendedStorage features.");
}
var storage = new BluepathStorage(bluepath.Storage as IExtendedStorage);
var reduceProvider = Loader.Load<IReduceProvider>(reduceFuncName, storage);
var reducer = new Reducer(filePath.ToString(), reduceProvider.Reduce, storage);
var reduceResult = reducer.PerformReduce();
Log.TraceMessage(Log.Activity.Custom,string.Format("[Reduce] Storing key '{0}', value '{1}'", reduceResult.Key, reduceResult.Value));
storage.Store(string.Format(Properties.Settings.Default.ReduceOutputFileName, reduceResult.Key, bluepath.ProcessEid, 0), reduceResult.Value);
return new List<string>() { reduceResult.Key };
});
return this.InitThread(startWorkerNo, workersCount, reduceFunc);
}
