Exemple #1
0
        public static void Reduce(IFileSource<string, string> inputfolders, IFileDestination<string, string> outputfolder, IReducer reducer, int threads)
        {
            var threadpool = new ThreadPool(threads, "Reducer file threads");

            Console.WriteLine("Preparing reducer threads.");
            string inputfileid;
            while (inputfolders.ReadNext(out inputfileid))
            {
                var ft = new ReduceFileThread();
                ft.InputFolders = inputfolders;
                ft.OutputFolder = outputfolder;
                // 	ft.Writer = wrt;
                ft.reducer = reducer;
                ft.FileID = inputfileid;
                threadpool.Queue(ft);
            }

            Console.WriteLine("Waiting for reducer threads to finish...");
            var t1 = new Timing("Inner mapper");
            threadpool.WaitAll();
            t1.End();
            Console.WriteLine("Reducer threads finished.");

            GC.Collect();
        }
Exemple #2
0
 internal MapAndReduceJob()
 {
     InputPartitioner = null;
     Mapper = null;
     Reducer = null;
     ShufflePartitioner = null;
     CombineOutput = false;
     Input = null;
     Output = null;
     InputFitsInMemory = false;
     MapperFitsInMemory = false;
     ShufflerFitsInMemory = false;
     ReducerFitsInMemory = false;
     RunReducerAfterEveryMapper = false;
 }
Exemple #3
0
        public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, IReducer prereducer, int threads)
        {
            // var partitioner = new StandardKeyPartitioner(Count);

            var threadpool = new ThreadPool(threads, "Mapper file threads");
            var threadpool2 = new ThreadPool(threads, "Mapper threads");

            Console.WriteLine("Preparing mappers...");

            string inputfileid;
            while (inputfolders.ReadNext(out inputfileid))
            {
                var ft = new MapFilesThread();
                ft.InputFolders = inputfolders;
                ft.OutputFolder = shuffleroutput;
                ft.mapper = mapper;
                ft.reducer = prereducer;
                ft.FileIDs.Add(inputfileid);
                ft.threadpool = threadpool;
                threadpool.Queue(ft);
                threadpool.Step();
            }

            Console.WriteLine("Waiting for mapper file threads to finish...");
            var t1 = new Timing("Inner mapper");
            threadpool.WaitAll();
            t1.End();
            Console.WriteLine("Mapper file threads finished.");
            /*
            Console.WriteLine("Waiting for mapper threads to finish...");
            var t2 = new Timing("Inner mapper");
            threadpool2.WaitAll();
            t2.End();
            Console.WriteLine("Mapper threads finished.");
            */
            GC.Collect();
        }
Exemple #4
0
 public static void Combine(IFileSource<string, string> inputs, IFileDestination<string, string> output)
 {
     var t0 = new Timing("Combining files");
     using (var writer = output.CreateWriter())
     {
         var stp = new ThreadPool(5);
         Console.WriteLine("Preparing combiners...");
         string inputfile;
         while (inputs.ReadNext(out inputfile))
         {
             stp.Queue(new CombinerThread
             {
                 outputwriter = writer,
                 filesource = inputs,
                 FileId = inputfile
             });
         }
         Console.WriteLine("Combining files...");
         stp.WaitAll();
     }
     t0.End();
     Console.WriteLine("Combining done.");
     GC.Collect();
 }
 public void To(IFileDestination<string, string> output)
 {
     job.CombineOutput = false;
     job.Output = output;
     job.Run();
 }
Exemple #6
0
 public static void Reduce(IFileSource<string, string> inputfolders, IFileDestination<string, string> outputfolder, IReducer reducer)
 {
     Reduce(inputfolders, outputfolder, reducer, 30);
 }
Exemple #7
0
 public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, int threads)
 {
     Map(inputfolders, shuffleroutput, mapper, null, threads);
 }
Exemple #8
0
 public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, IReducer prereducer)
 {
     Map(inputfolders, shuffleroutput, mapper, prereducer, 50);
 }