public void Run()
{
var TempRW = "Possan.MapReduce.IO.TabFileFolder, Possan.MapReduce";
// partition input... N outputs
Console.WriteLine("==============================================");
var partitioners = new ClientConnectionCollection();
var partitionconfig = new ClientConfig();
partitionconfig.Assemblies = Assemblies;
partitionconfig.ManagerUrl = ManagerUrl;
partitionconfig.Instances = 1;
partitionconfig.JobType = "Possan.MapReduce.Distributed.Jobs.ShuffleJob, Possan.MapReduce.Distributed.Jobs";
partitionconfig.JobArgs.Add("partitioner", InputPartitionerTypeName);
partitionconfig.JobArgs.Add("sort", "1");
partitionconfig.JobArgs.Add("input", InputType + "=" + InputFolder);
for (var k = 0; k < NumInputPartitions; k++)
partitionconfig.JobArgs.Add("output", TempRW + "=" + TempFolder + "\\partitioned-input-" + k);
partitioners.Add(partitionconfig);
partitioners.StartAllAndWait();
Console.WriteLine("==============================================");
// run N mappers+prereducers + join to N reducers
var mappers = new ClientConnectionCollection();
for (int k = 0; k < NumInputPartitions; k++)
{
var mapperconfig = new ClientConfig();
mapperconfig.Assemblies = Assemblies;
mapperconfig.ManagerUrl = ManagerUrl;
mapperconfig.Instances = 1;
mapperconfig.JobType = "Possan.MapReduce.Distributed.Jobs.MapJob, Possan.MapReduce.Distributed.Jobs";
mapperconfig.JobArgs.Add("mapper", MapperTypeName);
mapperconfig.JobArgs.Add("input", TempRW + "=" + TempFolder + "\\partitioned-input-" + k);
mapperconfig.JobArgs.Add("output", TempRW + "=" + TempFolder + "\\mapper-output-" + k);
mappers.Add(mapperconfig);
}
mappers.StartAllAndWait();
Console.WriteLine("==============================================");
// shuffle/sort to N reducers
var sorters = new ClientConnectionCollection();
for (int k = 0; k < NumInputPartitions; k++)
{
var sorterconfig = new ClientConfig();
sorterconfig.Assemblies = Assemblies;
sorterconfig.ManagerUrl = ManagerUrl;
sorterconfig.Instances = 1;
sorterconfig.JobType = "Possan.MapReduce.Distributed.Jobs.ShuffleJob, Possan.MapReduce.Distributed.Jobs";
sorterconfig.JobArgs.Add("partitioner", ShufflerPartitionerTypeName);
sorterconfig.JobArgs.Add("sort", "1");
sorterconfig.JobArgs.Add("input", TempRW + "=" + TempFolder + "\\mapper-output-" + k);
for (var u = 0; u < NumReducerPartitions; u++)
sorterconfig.JobArgs.Add("output", TempRW + "=" + TempFolder + "\\reducer-input-" + u);
sorters.Add(sorterconfig);
}
sorters.StartAllAndWait();
Console.WriteLine("==============================================");
// run N reducers
var reducers = new ClientConnectionCollection();
for (int k = 0; k < NumReducerPartitions; k++)
{
var reducerconfig = new ClientConfig();
reducerconfig.Assemblies = Assemblies;
reducerconfig.ManagerUrl = ManagerUrl;
reducerconfig.Instances = 1;
reducerconfig.JobType = "Possan.MapReduce.Distributed.Jobs.ReduceJob, Possan.MapReduce.Distributed.Jobs";
reducerconfig.JobArgs.Add("reducer", ReducerTypeName);
reducerconfig.JobArgs.Add("input", TempRW + "=" + TempFolder + "\\reducer-input-" + k);
reducerconfig.JobArgs.Add("output", TempRW + "=" + TempFolder + "\\reducer-output-" + k);
reducers.Add(reducerconfig);
}
reducers.StartAllAndWait();
Console.WriteLine("==============================================");
// combine
var combiners = new ClientConnectionCollection();
var combinerconfig = new ClientConfig();
combinerconfig.Assemblies = Assemblies;
combinerconfig.ManagerUrl = ManagerUrl;
combinerconfig.Instances = 1;
combinerconfig.JobType = "Possan.MapReduce.Distributed.Jobs.ShuffleJob, Possan.MapReduce.Distributed.Jobs";
combinerconfig.JobArgs.Add("partitioner", CombinePartitionerTypeName );
combinerconfig.JobArgs.Add("sort", "1");
for (var u = 0; u < NumReducerPartitions; u++)
combinerconfig.JobArgs.Add("input", TempRW + "=" + TempFolder + "\\reducer-output-" + u);
combinerconfig.JobArgs.Add("output", OutputType + "=" + OutputFolder );
combiners.Add(combinerconfig);
combiners.StartAllAndWait();
Console.WriteLine("==============================================");
}
public ClientConnection(ClientConfig cfg)
{
JobId = "";
_cfg = cfg;
State = ClientConnectionState.NotStarted;
}