예제 #1
0
        public static void reduce()
        {
            Assembly assembly    = Assembly.LoadFrom(MapReduceUtils.pathToDll);
            Type     reduceClass = assembly.GetTypes().Where(t => typeof(ApiMaperReducer.ApiReducer).IsAssignableFrom(t)).First();

            if (reduceClass == null)
            {
                throw new Exception("Reducer class not found!");
            }

            ApiMaperReducer.ApiReducer reducer = (ApiMaperReducer.ApiReducer)Activator.CreateInstance(reduceClass);

            String       filePath = DfsWorkerUtils.getPathToChunk(StatusConfigContainer.fileNameOut, StatusConfigContainer.workerId);
            StreamWriter writer   = new StreamWriter(filePath);

            reducer.writer = writer;

            foreach (var key in  keyList)
            {
                var keyFile = key.Value + MAPER_RESULT_EXTENSION;
                //var key = keyFileName.Substring (0, keyFileName.Length - MAPER_RESULT_EXTENSION.Length); //check it!!!
                List <String> values = System.IO.File.ReadAllLines(Path.Combine(MapReduceUtils.GetWorkingDirectory(), keyFile)).ToList();

                reducer.reduce(key.Key, values);
            }

            writer.Close();
            log.Info("Reducer ended its work!");
            StatusConfigContainer.Status = StatusType.END;
        }
예제 #2
0
 public object Any(PrepareMRTask request)
 {
     log.Info("Prepare MapReduce... ");
     MapReduceUtils.clearWorkingDirectory();
     MapReduceUtils.saveDll(request.fileWithDll);
     StatusConfigContainer.reset();
     StatusConfigContainer.prepare(request.fileNameIn, request.fileNameOut, request.taskAssigments);
     return(new PrepareMRTaskResponse());
 }
예제 #3
0
        public static void saveMapResult(SendMappedData request)
        {
            foreach (var keyValue in request.listOfMapperResults)
            {
                var path = Path.Combine(MapReduceUtils.GetWorkingDirectory(), getIdOfKey(keyValue.Key) + MAPER_RESULT_EXTENSION);
                SaveQueue.append(path, keyValue.Value);
            }
            //System.IO.File.AppendAllText (path, request.value + Environment.NewLine);

            //var path = Path.Combine (MapReduceUtils.GetWorkingDirectory (), getIdOfKey (request.key) + MAPER_RESULT_EXTENSION);
            //SaveQueue.append (path, request.value);
        }
예제 #4
0
        public static void map()
        {
            String dllPath = Path.Combine(MapReduceUtils.GetWorkingDirectory(), MapReduceUtils.USERDLL_NAME);

            log.Debug("Dll in  " + dllPath);

            Assembly assembly = Assembly.LoadFrom(dllPath);

            if (assembly == null)
            {
                throw new Exception("Assembly is empty!");
            }

//			AppDomain.CurrentDomain.Load(assembly.GetName());
            Type mapClass = assembly.GetTypes().Where(t => typeof(ApiMaperReducer.ApiMapper).IsAssignableFrom(t)).First();

            if (mapClass == null)
            {
                throw new Exception("Mapper class not found!");
            }

            ApiMaperReducer.ApiMapper mapper = (ApiMaperReducer.ApiMapper)Activator.CreateInstance(mapClass);
            mapper.setListOfNodes(StatusConfigContainer.reducersIps);

            foreach (int chunk in StatusConfigContainer.chunksToProcess)
            {
                log.InfoFormat("Mapping chunk {0}...", chunk);
                // Read the file and display it line by line.
                mapper.chunk = chunk;
                mapper.init(chunk);
                Chunk readedChunk = DfsWorkerUtils.readChunk(StatusConfigContainer.fileNameIn, chunk);

                foreach (string line in readedChunk.data)
                {
                    log.DebugFormat("Reading line: {0}", line);
                    mapper.map(line);
                }
                mapper.endWork();
            }

            log.Info("Mapper ended its work!");
            StatusConfigContainer.Status = StatusType.WAITING_FOR_REDUCE;
        }