public static void Reduce(IFileSource<string, string> inputfolders, IFileDestination<string, string> outputfolder, IReducer reducer, int threads) { var threadpool = new ThreadPool(threads, "Reducer file threads"); Console.WriteLine("Preparing reducer threads."); string inputfileid; while (inputfolders.ReadNext(out inputfileid)) { var ft = new ReduceFileThread(); ft.InputFolders = inputfolders; ft.OutputFolder = outputfolder; // ft.Writer = wrt; ft.reducer = reducer; ft.FileID = inputfileid; threadpool.Queue(ft); } Console.WriteLine("Waiting for reducer threads to finish..."); var t1 = new Timing("Inner mapper"); threadpool.WaitAll(); t1.End(); Console.WriteLine("Reducer threads finished."); GC.Collect(); }
public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, IReducer prereducer, int threads) { // var partitioner = new StandardKeyPartitioner(Count); var threadpool = new ThreadPool(threads, "Mapper file threads"); var threadpool2 = new ThreadPool(threads, "Mapper threads"); Console.WriteLine("Preparing mappers..."); string inputfileid; while (inputfolders.ReadNext(out inputfileid)) { var ft = new MapFilesThread(); ft.InputFolders = inputfolders; ft.OutputFolder = shuffleroutput; ft.mapper = mapper; ft.reducer = prereducer; ft.FileIDs.Add(inputfileid); ft.threadpool = threadpool; threadpool.Queue(ft); threadpool.Step(); } Console.WriteLine("Waiting for mapper file threads to finish..."); var t1 = new Timing("Inner mapper"); threadpool.WaitAll(); t1.End(); Console.WriteLine("Mapper file threads finished."); /* Console.WriteLine("Waiting for mapper threads to finish..."); var t2 = new Timing("Inner mapper"); threadpool2.WaitAll(); t2.End(); Console.WriteLine("Mapper threads finished."); */ GC.Collect(); }
public static void Combine(IFileSource<string, string> inputs, IFileDestination<string, string> output) { var t0 = new Timing("Combining files"); using (var writer = output.CreateWriter()) { var stp = new ThreadPool(5); Console.WriteLine("Preparing combiners..."); string inputfile; while (inputs.ReadNext(out inputfile)) { stp.Queue(new CombinerThread { outputwriter = writer, filesource = inputs, FileId = inputfile }); } Console.WriteLine("Combining files..."); stp.WaitAll(); } t0.End(); Console.WriteLine("Combining done."); GC.Collect(); }
static void Main(string[] args) { var timer = new Timing("Entire app"); int numpoints = 10000; var tempInput = new DummyFileSource(numpoints, 15, "key", "value"); var tempOutput = new InMemoryTempFolder(); Fluently.Map.Input(tempInput).WithInMemoryMapper(new RandomPointInCircleMapper()).ReduceInMemoryUsing(new SummaryReducer()).CombineTo(tempOutput); int num_inside = 0; string k, v; while (tempOutput.Buffer.Read(out k, out v)) { // Console.WriteLine(k + " >>> " + v); num_inside += int.Parse(v, CultureInfo.InvariantCulture); } double pi = num_inside * 4.0 / numpoints; Console.WriteLine("PI estimated to: " + pi + " (" + num_inside + " of " + numpoints + " inside circle)"); timer.End(); }
static void Main(string[] args) { var timer = new Timing("All"); Thread.Sleep(4000); /* int baseport = 7890; // set up manager var mgr = new Manager(new ManagerConfig { Port = baseport }); mgr.Start(); // set up workers var workers = new List<Worker>(); for (int i = 0; i < 20; i++) { var w = new Worker(new WorkerConfig { ManagerUrl = "http://127.0.0.1:" + baseport, Port = baseport + 1 + i, MaxThreads = 1 }); workers.Add(w); w.Start(); } */ /* var ccfg = new ClientConfig(); ccfg.Assemblies.Add("Possan.MapReduce.dll"); ccfg.Assemblies.Add("Possan.Distributed.Sandbox.dll"); ccfg.Assemblies.Add("ClientServer.Shared.dll"); ccfg.JobType = "ClientServer.Shared.TestJob, ClientServer.Shared"; ccfg.JobArgs.Add("test","hej"); ccfg.ManagerUrl = "http://127.0.0.1:8000"; ccfg.Instances = 1; var client = new ClientConnection(ccfg); client.Run(); ccfg = new ClientConfig(); ccfg.Assemblies.Add("Possan.Distributed.dll"); ccfg.Assemblies.Add("Possan.MapReduce.Distributed.dll"); ccfg.Assemblies.Add("ClientServer.Shared.dll"); ccfg.JobType = "ClientServer.Shared.TestJob, ClientServer.Shared"; ccfg.JobArgs = new List<string> { "a2", "a3", "a4" }; ccfg.ManagerUrl = "http://127.0.0.1:" + baseport; ccfg.Instances = 4; client = new ClientConnection(ccfg); client.Run(); */ var suffix = DateTime.Now.Ticks.ToString();// Guid.NewGuid().ToString().Replace("-", "").Substring(0, 10); var conn = new MapReduceConnection(); conn.ManagerUrl = "http://127.0.0.1:8000";// +baseport; conn.InputFolder = "c:\\temp\\smalltextfiles"; conn.InputType = "Possan.MapReduce.IO.TextFilesFolderSource, Possan.MapReduce"; conn.TempFolder = "c:\\temp\\MR4\\temp-" + suffix; conn.OutputFolder = "c:\\temp\\MR4\\output-" + suffix; conn.OutputType = "Possan.MapReduce.IO.TabFileFolder, Possan.MapReduce"; conn.Assemblies.Add("ClientServer.Shared.dll"); conn.Instances = 8; conn.NumInputPartitions = 5; conn.NumReducerPartitions = 4; conn.InputPartitionerTypeName = "Possan.MapReduce.Partitioners.MD5Partitioner, Possan.MapReduce"; conn.ShufflerPartitionerTypeName = "Possan.MapReduce.Partitioners.FirstCharacterPartitioner, Possan.MapReduce"; conn.CombinePartitionerTypeName = "Possan.MapReduce.Partitioners.FirstCharacterPartitioner, Possan.MapReduce"; conn.MapperTypeName = "ClientServer.Shared.TestMapper, ClientServer.Shared"; conn.ReducerTypeName = "ClientServer.Shared.TestReducer, ClientServer.Shared"; conn.Run(); /* // kill workers foreach (var w in workers) w.Stop(); // stop manager mgr.Stop(); */ timer.End(); Console.WriteLine("Hit [enter] to exit"); Console.ReadLine(); }