public static void Reduce(IFileSource<string, string> inputfolders, IFileDestination<string, string> outputfolder, IReducer reducer, int threads)
{
var threadpool = new ThreadPool(threads, "Reducer file threads");
Console.WriteLine("Preparing reducer threads.");
string inputfileid;
while (inputfolders.ReadNext(out inputfileid))
{
var ft = new ReduceFileThread();
ft.InputFolders = inputfolders;
ft.OutputFolder = outputfolder;
// ft.Writer = wrt;
ft.reducer = reducer;
ft.FileID = inputfileid;
threadpool.Queue(ft);
}
Console.WriteLine("Waiting for reducer threads to finish...");
var t1 = new Timing("Inner mapper");
threadpool.WaitAll();
t1.End();
Console.WriteLine("Reducer threads finished.");
GC.Collect();
}
internal MapAndReduceJob()
{
InputPartitioner = null;
Mapper = null;
Reducer = null;
ShufflePartitioner = null;
CombineOutput = false;
Input = null;
Output = null;
InputFitsInMemory = false;
MapperFitsInMemory = false;
ShufflerFitsInMemory = false;
ReducerFitsInMemory = false;
RunReducerAfterEveryMapper = false;
}
public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, IReducer prereducer, int threads)
{
// var partitioner = new StandardKeyPartitioner(Count);
var threadpool = new ThreadPool(threads, "Mapper file threads");
var threadpool2 = new ThreadPool(threads, "Mapper threads");
Console.WriteLine("Preparing mappers...");
string inputfileid;
while (inputfolders.ReadNext(out inputfileid))
{
var ft = new MapFilesThread();
ft.InputFolders = inputfolders;
ft.OutputFolder = shuffleroutput;
ft.mapper = mapper;
ft.reducer = prereducer;
ft.FileIDs.Add(inputfileid);
ft.threadpool = threadpool;
threadpool.Queue(ft);
threadpool.Step();
}
Console.WriteLine("Waiting for mapper file threads to finish...");
var t1 = new Timing("Inner mapper");
threadpool.WaitAll();
t1.End();
Console.WriteLine("Mapper file threads finished.");
/*
Console.WriteLine("Waiting for mapper threads to finish...");
var t2 = new Timing("Inner mapper");
threadpool2.WaitAll();
t2.End();
Console.WriteLine("Mapper threads finished.");
*/
GC.Collect();
}
public static void Combine(IFileSource<string, string> inputs, IFileDestination<string, string> output)
{
var t0 = new Timing("Combining files");
using (var writer = output.CreateWriter())
{
var stp = new ThreadPool(5);
Console.WriteLine("Preparing combiners...");
string inputfile;
while (inputs.ReadNext(out inputfile))
{
stp.Queue(new CombinerThread
{
outputwriter = writer,
filesource = inputs,
FileId = inputfile
});
}
Console.WriteLine("Combining files...");
stp.WaitAll();
}
t0.End();
Console.WriteLine("Combining done.");
GC.Collect();
}
public void To(IFileDestination<string, string> output)
{
job.CombineOutput = false;
job.Output = output;
job.Run();
}
public static void Reduce(IFileSource<string, string> inputfolders, IFileDestination<string, string> outputfolder, IReducer reducer)
{
Reduce(inputfolders, outputfolder, reducer, 30);
}
public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, int threads)
{
Map(inputfolders, shuffleroutput, mapper, null, threads);
}
public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, IReducer prereducer)
{
Map(inputfolders, shuffleroutput, mapper, prereducer, 50);
}
