public static void reduce()
{
Assembly assembly = Assembly.LoadFrom(MapReduceUtils.pathToDll);
Type reduceClass = assembly.GetTypes().Where(t => typeof(ApiMaperReducer.ApiReducer).IsAssignableFrom(t)).First();
if (reduceClass == null)
{
throw new Exception("Reducer class not found!");
}
ApiMaperReducer.ApiReducer reducer = (ApiMaperReducer.ApiReducer)Activator.CreateInstance(reduceClass);
String filePath = DfsWorkerUtils.getPathToChunk(StatusConfigContainer.fileNameOut, StatusConfigContainer.workerId);
StreamWriter writer = new StreamWriter(filePath);
reducer.writer = writer;
foreach (var key in keyList)
{
var keyFile = key.Value + MAPER_RESULT_EXTENSION;
//var key = keyFileName.Substring (0, keyFileName.Length - MAPER_RESULT_EXTENSION.Length); //check it!!!
List <String> values = System.IO.File.ReadAllLines(Path.Combine(MapReduceUtils.GetWorkingDirectory(), keyFile)).ToList();
reducer.reduce(key.Key, values);
}
writer.Close();
log.Info("Reducer ended its work!");
StatusConfigContainer.Status = StatusType.END;
}
public static void deleteAllChunksOfFile(String name)
{
var fileChunks = (from chunk in DfsWorkerUtils.listWorkingDirectory()
where chunk.fileName == name
select chunk).ToList();
if (fileChunks.Count == 0)
{
throw new ArgumentException("No chunk of file exists on this worker!");
}
foreach (var chunks in fileChunks)
{
DfsWorkerUtils.deleteChunk(name, chunks.chunkId);
}
}
public object Delete(DeleteChunk request)
{
if (request.chunkId < -1)
{
throw new Exception("Chunk ID can not be negative! (Eventually use -1 to delete all chunks)");
}
if (request.chunkId == -1) //delete all chunks of file
{
DfsWorkerUtils.deleteAllChunksOfFile(request.FileName);
}
else
{
DfsWorkerUtils.deleteChunk(request.FileName, request.chunkId);
}
return(true);
}
public static void map()
{
String dllPath = Path.Combine(MapReduceUtils.GetWorkingDirectory(), MapReduceUtils.USERDLL_NAME);
log.Debug("Dll in " + dllPath);
Assembly assembly = Assembly.LoadFrom(dllPath);
if (assembly == null)
{
throw new Exception("Assembly is empty!");
}
// AppDomain.CurrentDomain.Load(assembly.GetName());
Type mapClass = assembly.GetTypes().Where(t => typeof(ApiMaperReducer.ApiMapper).IsAssignableFrom(t)).First();
if (mapClass == null)
{
throw new Exception("Mapper class not found!");
}
ApiMaperReducer.ApiMapper mapper = (ApiMaperReducer.ApiMapper)Activator.CreateInstance(mapClass);
mapper.setListOfNodes(StatusConfigContainer.reducersIps);
foreach (int chunk in StatusConfigContainer.chunksToProcess)
{
log.InfoFormat("Mapping chunk {0}...", chunk);
// Read the file and display it line by line.
mapper.chunk = chunk;
mapper.init(chunk);
Chunk readedChunk = DfsWorkerUtils.readChunk(StatusConfigContainer.fileNameIn, chunk);
foreach (string line in readedChunk.data)
{
log.DebugFormat("Reading line: {0}", line);
mapper.map(line);
}
mapper.endWork();
}
log.Info("Mapper ended its work!");
StatusConfigContainer.Status = StatusType.WAITING_FOR_REDUCE;
}
public static void Main(string[] args)
{
Console.WriteLine("Hello World!");
Console.WriteLine("Remember IP must be IDENTICAL as in master config!");
DfsWorkerUtils.createWorkingDirectory();
var listeningOn = args.Length == 0 ? "http://*:1337/" : args [0];
if (!listeningOn.Contains("http"))
{
Console.WriteLine("Start IP with http(s)");
}
if (!listeningOn.EndsWith("/"))
{
Console.WriteLine("End IP with /");
}
var appHost = new WorkerHost().Init().Start(listeningOn);
StatusConfigContainer.init(listeningOn);
Console.WriteLine("WorkerHost Created at {0}, listening on {1}",
DateTime.Now, listeningOn);
Console.ReadKey();
}
public object Any(Ls request)
{
return(new LsResponse {
Result = DfsWorkerUtils.listWorkingDirectory()
});
}
public object Put(SaveChunk request)
{
DfsWorkerUtils.writeChunk(request.chunk);
return(true);
}
public object Get(GetChunk request)
{
return(new GetChunkResponse {
Result = DfsWorkerUtils.readChunk(request.FileName, request.chunkId)
});
}