public override bool TryExecuteTask()
{
bool taskAvailable = db.AssignReduceTask();
if (!taskAvailable)
{
return(false);
}
var task = db.GetReduceTask();
IMapReduce mapReduce = db.LoadAssembly();
Reducer reducer = new Reducer(mapReduce);
var result = reducer.Reduce(task.Key, task.Value);
db.FinishReduceTask(result, ReduceDone);
return(true);
}
public override bool TryExecuteTask()
{
bool taskAvailable = db.AssignMapTask();
if (!taskAvailable)
{
return(false);
}
Stream input = db.GetDataStream();
IMapReduce mapReduce = db.LoadAssembly();
Mapper mapper = new Mapper(mapReduce);
mapper.ReadAndMap(input, db.KeyValueDataCollector);
input.Close();
db.FinishMapTask(MapDone);
return(true);
}
public static void Process <TKey, TValue>(this IMapReduce <TKey, TValue> mapreduce, params string[] args)
{
var source = args[0];
var target = args[1];
try {
if (args.Length == 2)
{
var map = mapreduce.Map(source);
var json = JsonConvert.SerializeObject(map);
File.WriteAllText(Path.Combine(target, System.Diagnostics.Process.GetCurrentProcess().Id.ToString()), json);
}
if (args.Length == 3)
{
var maps = new List <Dictionary <TKey, TValue> >();
foreach (var file in Directory.GetFiles(source))
{
var map = JsonConvert.DeserializeObject <Dictionary <TKey, TValue> >(File.ReadAllText(file));
maps.Add(map);
//File.Delete(file);
}
var result = mapreduce.Reduce(maps);
var json = JsonConvert.SerializeObject(result);
File.WriteAllText(Path.Combine(target, "out.json"), json);
}
}
catch (Exception ex)
{
var sb = new StringBuilder();
sb.AppendLine(ex.Message);
sb.AppendLine(ex.StackTrace);
File.WriteAllText(Path.Combine(target, "error.json"), sb.ToString());
throw;
}
}
public Mapper(IMapReduce implementation)
{
mapReduce = implementation;
dataCollector = new ConcurrentBag <KeyValuePair <byte[], byte[]> >();
}
public Reducer(IMapReduce implementation)
{
mapReduce = implementation;
}