public static void Reduce(IFileSource<string, string> inputfolders, IFileDestination<string, string> outputfolder, IReducer reducer, int threads)
{
var threadpool = new ThreadPool(threads, "Reducer file threads");
Console.WriteLine("Preparing reducer threads.");
string inputfileid;
while (inputfolders.ReadNext(out inputfileid))
{
var ft = new ReduceFileThread();
ft.InputFolders = inputfolders;
ft.OutputFolder = outputfolder;
// ft.Writer = wrt;
ft.reducer = reducer;
ft.FileID = inputfileid;
threadpool.Queue(ft);
}
Console.WriteLine("Waiting for reducer threads to finish...");
var t1 = new Timing("Inner mapper");
threadpool.WaitAll();
t1.End();
Console.WriteLine("Reducer threads finished.");
GC.Collect();
}
public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, IReducer prereducer, int threads)
{
// var partitioner = new StandardKeyPartitioner(Count);
var threadpool = new ThreadPool(threads, "Mapper file threads");
var threadpool2 = new ThreadPool(threads, "Mapper threads");
Console.WriteLine("Preparing mappers...");
string inputfileid;
while (inputfolders.ReadNext(out inputfileid))
{
var ft = new MapFilesThread();
ft.InputFolders = inputfolders;
ft.OutputFolder = shuffleroutput;
ft.mapper = mapper;
ft.reducer = prereducer;
ft.FileIDs.Add(inputfileid);
ft.threadpool = threadpool;
threadpool.Queue(ft);
threadpool.Step();
}
Console.WriteLine("Waiting for mapper file threads to finish...");
var t1 = new Timing("Inner mapper");
threadpool.WaitAll();
t1.End();
Console.WriteLine("Mapper file threads finished.");
/*
Console.WriteLine("Waiting for mapper threads to finish...");
var t2 = new Timing("Inner mapper");
threadpool2.WaitAll();
t2.End();
Console.WriteLine("Mapper threads finished.");
*/
GC.Collect();
}
public static void Combine(IFileSource<string, string> inputs, IFileDestination<string, string> output)
{
var t0 = new Timing("Combining files");
using (var writer = output.CreateWriter())
{
var stp = new ThreadPool(5);
Console.WriteLine("Preparing combiners...");
string inputfile;
while (inputs.ReadNext(out inputfile))
{
stp.Queue(new CombinerThread
{
outputwriter = writer,
filesource = inputs,
FileId = inputfile
});
}
Console.WriteLine("Combining files...");
stp.WaitAll();
}
t0.End();
Console.WriteLine("Combining done.");
GC.Collect();
}
static void Main(string[] args)
{
var timer = new Timing("Entire app");
int numpoints = 10000;
var tempInput = new DummyFileSource(numpoints, 15, "key", "value");
var tempOutput = new InMemoryTempFolder();
Fluently.Map.Input(tempInput).WithInMemoryMapper(new RandomPointInCircleMapper()).ReduceInMemoryUsing(new SummaryReducer()).CombineTo(tempOutput);
int num_inside = 0;
string k, v;
while (tempOutput.Buffer.Read(out k, out v))
{
// Console.WriteLine(k + " >>> " + v);
num_inside += int.Parse(v, CultureInfo.InvariantCulture);
}
double pi = num_inside * 4.0 / numpoints;
Console.WriteLine("PI estimated to: " + pi + " (" + num_inside + " of " + numpoints + " inside circle)");
timer.End();
}
static void Main(string[] args)
{
var timer = new Timing("All");
Thread.Sleep(4000);
/*
int baseport = 7890;
// set up manager
var mgr = new Manager(new ManagerConfig { Port = baseport });
mgr.Start();
// set up workers
var workers = new List<Worker>();
for (int i = 0; i < 20; i++)
{
var w = new Worker(new WorkerConfig { ManagerUrl = "http://127.0.0.1:" + baseport, Port = baseport + 1 + i, MaxThreads = 1 });
workers.Add(w);
w.Start();
}
*/
/*
var ccfg = new ClientConfig();
ccfg.Assemblies.Add("Possan.MapReduce.dll");
ccfg.Assemblies.Add("Possan.Distributed.Sandbox.dll");
ccfg.Assemblies.Add("ClientServer.Shared.dll");
ccfg.JobType = "ClientServer.Shared.TestJob, ClientServer.Shared";
ccfg.JobArgs.Add("test","hej");
ccfg.ManagerUrl = "http://127.0.0.1:8000";
ccfg.Instances = 1;
var client = new ClientConnection(ccfg);
client.Run();
ccfg = new ClientConfig();
ccfg.Assemblies.Add("Possan.Distributed.dll");
ccfg.Assemblies.Add("Possan.MapReduce.Distributed.dll");
ccfg.Assemblies.Add("ClientServer.Shared.dll");
ccfg.JobType = "ClientServer.Shared.TestJob, ClientServer.Shared";
ccfg.JobArgs = new List<string> { "a2", "a3", "a4" };
ccfg.ManagerUrl = "http://127.0.0.1:" + baseport;
ccfg.Instances = 4;
client = new ClientConnection(ccfg);
client.Run();
*/
var suffix = DateTime.Now.Ticks.ToString();// Guid.NewGuid().ToString().Replace("-", "").Substring(0, 10);
var conn = new MapReduceConnection();
conn.ManagerUrl = "http://127.0.0.1:8000";// +baseport;
conn.InputFolder = "c:\\temp\\smalltextfiles";
conn.InputType = "Possan.MapReduce.IO.TextFilesFolderSource, Possan.MapReduce";
conn.TempFolder = "c:\\temp\\MR4\\temp-" + suffix;
conn.OutputFolder = "c:\\temp\\MR4\\output-" + suffix;
conn.OutputType = "Possan.MapReduce.IO.TabFileFolder, Possan.MapReduce";
conn.Assemblies.Add("ClientServer.Shared.dll");
conn.Instances = 8;
conn.NumInputPartitions = 5;
conn.NumReducerPartitions = 4;
conn.InputPartitionerTypeName = "Possan.MapReduce.Partitioners.MD5Partitioner, Possan.MapReduce";
conn.ShufflerPartitionerTypeName = "Possan.MapReduce.Partitioners.FirstCharacterPartitioner, Possan.MapReduce";
conn.CombinePartitionerTypeName = "Possan.MapReduce.Partitioners.FirstCharacterPartitioner, Possan.MapReduce";
conn.MapperTypeName = "ClientServer.Shared.TestMapper, ClientServer.Shared";
conn.ReducerTypeName = "ClientServer.Shared.TestReducer, ClientServer.Shared";
conn.Run();
/*
// kill workers
foreach (var w in workers)
w.Stop();
// stop manager
mgr.Stop();
*/
timer.End();
Console.WriteLine("Hit [enter] to exit");
Console.ReadLine();
}