public static void reduce()
{
Assembly assembly = Assembly.LoadFrom(MapReduceUtils.pathToDll);
Type reduceClass = assembly.GetTypes().Where(t => typeof(ApiMaperReducer.ApiReducer).IsAssignableFrom(t)).First();
if (reduceClass == null)
{
throw new Exception("Reducer class not found!");
}
ApiMaperReducer.ApiReducer reducer = (ApiMaperReducer.ApiReducer)Activator.CreateInstance(reduceClass);
String filePath = DfsWorkerUtils.getPathToChunk(StatusConfigContainer.fileNameOut, StatusConfigContainer.workerId);
StreamWriter writer = new StreamWriter(filePath);
reducer.writer = writer;
foreach (var key in keyList)
{
var keyFile = key.Value + MAPER_RESULT_EXTENSION;
//var key = keyFileName.Substring (0, keyFileName.Length - MAPER_RESULT_EXTENSION.Length); //check it!!!
List <String> values = System.IO.File.ReadAllLines(Path.Combine(MapReduceUtils.GetWorkingDirectory(), keyFile)).ToList();
reducer.reduce(key.Key, values);
}
writer.Close();
log.Info("Reducer ended its work!");
StatusConfigContainer.Status = StatusType.END;
}
public object Any(PrepareMRTask request)
{
log.Info("Prepare MapReduce... ");
MapReduceUtils.clearWorkingDirectory();
MapReduceUtils.saveDll(request.fileWithDll);
StatusConfigContainer.reset();
StatusConfigContainer.prepare(request.fileNameIn, request.fileNameOut, request.taskAssigments);
return(new PrepareMRTaskResponse());
}
public static void saveMapResult(SendMappedData request)
{
foreach (var keyValue in request.listOfMapperResults)
{
var path = Path.Combine(MapReduceUtils.GetWorkingDirectory(), getIdOfKey(keyValue.Key) + MAPER_RESULT_EXTENSION);
SaveQueue.append(path, keyValue.Value);
}
//System.IO.File.AppendAllText (path, request.value + Environment.NewLine);
//var path = Path.Combine (MapReduceUtils.GetWorkingDirectory (), getIdOfKey (request.key) + MAPER_RESULT_EXTENSION);
//SaveQueue.append (path, request.value);
}
public static void map()
{
String dllPath = Path.Combine(MapReduceUtils.GetWorkingDirectory(), MapReduceUtils.USERDLL_NAME);
log.Debug("Dll in " + dllPath);
Assembly assembly = Assembly.LoadFrom(dllPath);
if (assembly == null)
{
throw new Exception("Assembly is empty!");
}
// AppDomain.CurrentDomain.Load(assembly.GetName());
Type mapClass = assembly.GetTypes().Where(t => typeof(ApiMaperReducer.ApiMapper).IsAssignableFrom(t)).First();
if (mapClass == null)
{
throw new Exception("Mapper class not found!");
}
ApiMaperReducer.ApiMapper mapper = (ApiMaperReducer.ApiMapper)Activator.CreateInstance(mapClass);
mapper.setListOfNodes(StatusConfigContainer.reducersIps);
foreach (int chunk in StatusConfigContainer.chunksToProcess)
{
log.InfoFormat("Mapping chunk {0}...", chunk);
// Read the file and display it line by line.
mapper.chunk = chunk;
mapper.init(chunk);
Chunk readedChunk = DfsWorkerUtils.readChunk(StatusConfigContainer.fileNameIn, chunk);
foreach (string line in readedChunk.data)
{
log.DebugFormat("Reading line: {0}", line);
mapper.map(line);
}
mapper.endWork();
}
log.Info("Mapper ended its work!");
StatusConfigContainer.Status = StatusType.WAITING_FOR_REDUCE;
}