public static void Reduce(IFileSource<string, string> inputfolders, IFileDestination<string, string> outputfolder, IReducer reducer, int threads)
{
var threadpool = new ThreadPool(threads, "Reducer file threads");
Console.WriteLine("Preparing reducer threads.");
string inputfileid;
while (inputfolders.ReadNext(out inputfileid))
{
var ft = new ReduceFileThread();
ft.InputFolders = inputfolders;
ft.OutputFolder = outputfolder;
// ft.Writer = wrt;
ft.reducer = reducer;
ft.FileID = inputfileid;
threadpool.Queue(ft);
}
Console.WriteLine("Waiting for reducer threads to finish...");
var t1 = new Timing("Inner mapper");
threadpool.WaitAll();
t1.End();
Console.WriteLine("Reducer threads finished.");
GC.Collect();
}
public void Dump(IFileSource<string, string> filesource, string title)
{
DumpHeader(title);
var fn = "";
while (filesource.ReadNext(out fn))
{
var reader = filesource.CreateStreamReader(fn);
string k, v;
while (reader.Read(out k, out v))
WriteLine(k + " => " + v);
}
DumpFooter(title);
}
public static void Map(IFileSource<string, string> inputfolders, IFileDestination<string, string> shuffleroutput, IMapper mapper, IReducer prereducer, int threads)
{
// var partitioner = new StandardKeyPartitioner(Count);
var threadpool = new ThreadPool(threads, "Mapper file threads");
var threadpool2 = new ThreadPool(threads, "Mapper threads");
Console.WriteLine("Preparing mappers...");
string inputfileid;
while (inputfolders.ReadNext(out inputfileid))
{
var ft = new MapFilesThread();
ft.InputFolders = inputfolders;
ft.OutputFolder = shuffleroutput;
ft.mapper = mapper;
ft.reducer = prereducer;
ft.FileIDs.Add(inputfileid);
ft.threadpool = threadpool;
threadpool.Queue(ft);
threadpool.Step();
}
Console.WriteLine("Waiting for mapper file threads to finish...");
var t1 = new Timing("Inner mapper");
threadpool.WaitAll();
t1.End();
Console.WriteLine("Mapper file threads finished.");
/*
Console.WriteLine("Waiting for mapper threads to finish...");
var t2 = new Timing("Inner mapper");
threadpool2.WaitAll();
t2.End();
Console.WriteLine("Mapper threads finished.");
*/
GC.Collect();
}
public static void Combine(IFileSource<string, string> inputs, IFileDestination<string, string> output)
{
var t0 = new Timing("Combining files");
using (var writer = output.CreateWriter())
{
var stp = new ThreadPool(5);
Console.WriteLine("Preparing combiners...");
string inputfile;
while (inputs.ReadNext(out inputfile))
{
stp.Queue(new CombinerThread
{
outputwriter = writer,
filesource = inputs,
FileId = inputfile
});
}
Console.WriteLine("Combining files...");
stp.WaitAll();
}
t0.End();
Console.WriteLine("Combining done.");
GC.Collect();
}