Exemplo n.º 1
0
        public static void Reduce(IFileSource<string, string> inputfolders, IFileDestination<string, string> outputfolder, IReducer reducer, int threads)
        {
            var threadpool = new ThreadPool(threads, "Reducer file threads");

            Console.WriteLine("Preparing reducer threads.");
            string inputfileid;
            while (inputfolders.ReadNext(out inputfileid))
            {
                var ft = new ReduceFileThread();
                ft.InputFolders = inputfolders;
                ft.OutputFolder = outputfolder;
                // 	ft.Writer = wrt;
                ft.reducer = reducer;
                ft.FileID = inputfileid;
                threadpool.Queue(ft);
            }

            Console.WriteLine("Waiting for reducer threads to finish...");
            var t1 = new Timing("Inner mapper");
            threadpool.WaitAll();
            t1.End();
            Console.WriteLine("Reducer threads finished.");

            GC.Collect();
        }
Exemplo n.º 2
0
        public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, IReducer prereducer, int threads)
        {
            // var partitioner = new StandardKeyPartitioner(Count);

            var threadpool = new ThreadPool(threads, "Mapper file threads");
            var threadpool2 = new ThreadPool(threads, "Mapper threads");

            Console.WriteLine("Preparing mappers...");

            string inputfileid;
            while (inputfolders.ReadNext(out inputfileid))
            {
                var ft = new MapFilesThread();
                ft.InputFolders = inputfolders;
                ft.OutputFolder = shuffleroutput;
                ft.mapper = mapper;
                ft.reducer = prereducer;
                ft.FileIDs.Add(inputfileid);
                ft.threadpool = threadpool;
                threadpool.Queue(ft);
                threadpool.Step();
            }

            Console.WriteLine("Waiting for mapper file threads to finish...");
            var t1 = new Timing("Inner mapper");
            threadpool.WaitAll();
            t1.End();
            Console.WriteLine("Mapper file threads finished.");
            /*
            Console.WriteLine("Waiting for mapper threads to finish...");
            var t2 = new Timing("Inner mapper");
            threadpool2.WaitAll();
            t2.End();
            Console.WriteLine("Mapper threads finished.");
            */
            GC.Collect();
        }
Exemplo n.º 3
0
 public static void Combine(IFileSource<string, string> inputs, IFileDestination<string, string> output)
 {
     var t0 = new Timing("Combining files");
     using (var writer = output.CreateWriter())
     {
         var stp = new ThreadPool(5);
         Console.WriteLine("Preparing combiners...");
         string inputfile;
         while (inputs.ReadNext(out inputfile))
         {
             stp.Queue(new CombinerThread
             {
                 outputwriter = writer,
                 filesource = inputs,
                 FileId = inputfile
             });
         }
         Console.WriteLine("Combining files...");
         stp.WaitAll();
     }
     t0.End();
     Console.WriteLine("Combining done.");
     GC.Collect();
 }
Exemplo n.º 4
0
        static void Main(string[] args)
        {
            var timer = new Timing("Entire app");

            int numpoints = 10000;

            var tempInput = new DummyFileSource(numpoints, 15, "key", "value");
            var tempOutput = new InMemoryTempFolder();

            Fluently.Map.Input(tempInput).WithInMemoryMapper(new RandomPointInCircleMapper()).ReduceInMemoryUsing(new SummaryReducer()).CombineTo(tempOutput);

            int num_inside = 0;
            string k, v;
            while (tempOutput.Buffer.Read(out k, out v))
            {
                // Console.WriteLine(k + " >>> " + v);
                num_inside += int.Parse(v, CultureInfo.InvariantCulture);
            }
            double pi = num_inside * 4.0 / numpoints;

            Console.WriteLine("PI estimated to: " + pi + " (" + num_inside + " of " + numpoints + " inside circle)");

            timer.End();
        }
Exemplo n.º 5
0
        static void Main(string[] args)
        {
            var timer = new Timing("All");

            Thread.Sleep(4000);

            /*
            int baseport = 7890;
            // set up manager
            var mgr = new Manager(new ManagerConfig { Port = baseport });
            mgr.Start();
            // set up workers
            var workers = new List<Worker>();
            for (int i = 0; i < 20; i++)
            {
                var w = new Worker(new WorkerConfig { ManagerUrl = "http://127.0.0.1:" + baseport, Port = baseport + 1 + i, MaxThreads = 1 });
                workers.Add(w);
                w.Start();
            }
            */

            /*
            var ccfg = new ClientConfig();
            ccfg.Assemblies.Add("Possan.MapReduce.dll");
            ccfg.Assemblies.Add("Possan.Distributed.Sandbox.dll");
            ccfg.Assemblies.Add("ClientServer.Shared.dll");
            ccfg.JobType = "ClientServer.Shared.TestJob, ClientServer.Shared";
            ccfg.JobArgs.Add("test","hej");
            ccfg.ManagerUrl = "http://127.0.0.1:8000";
            ccfg.Instances = 1;
            var client = new ClientConnection(ccfg);
            client.Run();

            ccfg = new ClientConfig();
            ccfg.Assemblies.Add("Possan.Distributed.dll");
            ccfg.Assemblies.Add("Possan.MapReduce.Distributed.dll");
            ccfg.Assemblies.Add("ClientServer.Shared.dll");
            ccfg.JobType = "ClientServer.Shared.TestJob, ClientServer.Shared";
            ccfg.JobArgs = new List<string> { "a2", "a3", "a4" };
            ccfg.ManagerUrl = "http://127.0.0.1:" + baseport;
            ccfg.Instances = 4;
            client = new ClientConnection(ccfg);
            client.Run();
            */

            var suffix = DateTime.Now.Ticks.ToString();// Guid.NewGuid().ToString().Replace("-", "").Substring(0, 10);
            var conn = new MapReduceConnection();
            conn.ManagerUrl = "http://127.0.0.1:8000";// +baseport;
            conn.InputFolder = "c:\\temp\\smalltextfiles";
            conn.InputType = "Possan.MapReduce.IO.TextFilesFolderSource, Possan.MapReduce";
            conn.TempFolder = "c:\\temp\\MR4\\temp-" + suffix;
            conn.OutputFolder = "c:\\temp\\MR4\\output-" + suffix;
            conn.OutputType = "Possan.MapReduce.IO.TabFileFolder, Possan.MapReduce";
            conn.Assemblies.Add("ClientServer.Shared.dll");
            conn.Instances = 8;
            conn.NumInputPartitions = 5;
            conn.NumReducerPartitions = 4;
            conn.InputPartitionerTypeName = "Possan.MapReduce.Partitioners.MD5Partitioner, Possan.MapReduce";
            conn.ShufflerPartitionerTypeName = "Possan.MapReduce.Partitioners.FirstCharacterPartitioner, Possan.MapReduce";
            conn.CombinePartitionerTypeName = "Possan.MapReduce.Partitioners.FirstCharacterPartitioner, Possan.MapReduce";
            conn.MapperTypeName = "ClientServer.Shared.TestMapper, ClientServer.Shared";
            conn.ReducerTypeName = "ClientServer.Shared.TestReducer, ClientServer.Shared";
            conn.Run();

            /*
            // kill workers
            foreach (var w in workers)
                w.Stop();
            // stop manager
            mgr.Stop();
            */
            timer.End();

            Console.WriteLine("Hit [enter] to exit");
            Console.ReadLine();
        }