/// <summary>
/// Runs the actual broadcast and reduce job
/// </summary>
public void Run(int numberofMappers, int chunkSize, int numIterations, int dim, int mapperMemory, int updateTaskMemory)
{
var updateFunctionConfig =
TangFactory.GetTang().NewConfigurationBuilder(IMRUUpdateConfiguration <int[], int[], int[]> .ConfigurationModule
.Set(IMRUUpdateConfiguration <int[], int[], int[]> .UpdateFunction,
GenericType <BroadcastSenderReduceReceiverUpdateFunction> .Class).Build())
.BindNamedParameter(typeof(BroadcastReduceConfiguration.NumberOfIterations),
numIterations.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(BroadcastReduceConfiguration.Dimensions),
dim.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(BroadcastReduceConfiguration.NumWorkers),
numberofMappers.ToString(CultureInfo.InvariantCulture))
.Build();
var dataConverterConfig1 =
TangFactory.GetTang()
.NewConfigurationBuilder(IMRUPipelineDataConverterConfiguration <int[]> .ConfigurationModule
.Set(IMRUPipelineDataConverterConfiguration <int[]> .MapInputPiplelineDataConverter,
GenericType <PipelineIntDataConverter> .Class).Build())
.BindNamedParameter(typeof(BroadcastReduceConfiguration.ChunkSize),
chunkSize.ToString(CultureInfo.InvariantCulture))
.Build();
var dataConverterConfig2 =
TangFactory.GetTang()
.NewConfigurationBuilder(IMRUPipelineDataConverterConfiguration <int[]> .ConfigurationModule
.Set(IMRUPipelineDataConverterConfiguration <int[]> .MapInputPiplelineDataConverter,
GenericType <PipelineIntDataConverter> .Class).Build())
.BindNamedParameter(typeof(BroadcastReduceConfiguration.ChunkSize),
chunkSize.ToString(CultureInfo.InvariantCulture))
.Build();
var results = _imruClient.Submit <int[], int[], int[]>(
new IMRUJobDefinitionBuilder()
.SetMapFunctionConfiguration(IMRUMapConfiguration <int[], int[]> .ConfigurationModule
.Set(IMRUMapConfiguration <int[], int[]> .MapFunction,
GenericType <BroadcastReceiverReduceSenderMapFunction> .Class)
.Build())
.SetUpdateFunctionConfiguration(updateFunctionConfig)
.SetMapInputCodecConfiguration(IMRUCodecConfiguration <int[]> .ConfigurationModule
.Set(IMRUCodecConfiguration <int[]> .Codec, GenericType <IntArrayStreamingCodec> .Class)
.Build())
.SetUpdateFunctionCodecsConfiguration(IMRUCodecConfiguration <int[]> .ConfigurationModule
.Set(IMRUCodecConfiguration <int[]> .Codec, GenericType <IntArrayStreamingCodec> .Class)
.Build())
.SetReduceFunctionConfiguration(IMRUReduceFunctionConfiguration <int[]> .ConfigurationModule
.Set(IMRUReduceFunctionConfiguration <int[]> .ReduceFunction,
GenericType <IntArraySumReduceFunction> .Class)
.Build())
.SetMapInputPipelineDataConverterConfiguration(dataConverterConfig1)
.SetMapOutputPipelineDataConverterConfiguration(dataConverterConfig2)
.SetPartitionedDatasetConfiguration(
RandomInputDataConfiguration.ConfigurationModule.Set(RandomInputDataConfiguration.NumberOfPartitions,
numberofMappers.ToString()).Build())
.SetJobName("BroadcastReduce")
.SetNumberOfMappers(numberofMappers)
.SetMapperMemory(mapperMemory)
.SetUpdateTaskMemory(updateTaskMemory)
.Build());
}
/// <summary>
/// Runs the actual broadcast and reduce job
/// </summary>
public void Run(int numberofMappers, int chunkSize, int numIterations, int dim, int mapperMemory, int updateTaskMemory)
{
var results = _imruClient.Submit <int[], int[], int[], Stream>(
CreateJobDefinitionBuilder(numberofMappers, chunkSize, numIterations, dim, mapperMemory, updateTaskMemory)
.SetMapFunctionConfiguration(BuildMapperFunctionConfig())
.Build());
}
/// <summary>
/// Runs the actual natural sum job.
/// </summary>
/// <returns>The result of the natural sum IMRU job.</returns>
public int Run(int numberofMappers, string outputFile, IConfiguration fileSystemConfig)
{
var results = _imruClient.Submit <int, int, int, Stream>(
new IMRUJobDefinitionBuilder()
.SetMapFunctionConfiguration(IMRUMapConfiguration <int, int> .ConfigurationModule
.Set(IMRUMapConfiguration <int, int> .MapFunction, GenericType <NaturalSumMapFunction> .Class)
.Build())
.SetUpdateFunctionConfiguration(
IMRUUpdateConfiguration <int, int, int> .ConfigurationModule
.Set(IMRUUpdateConfiguration <int, int, int> .UpdateFunction,
GenericType <SingleIterUpdateFunction> .Class)
.Build())
.SetMapInputCodecConfiguration(IMRUCodecConfiguration <int> .ConfigurationModule
.Set(IMRUCodecConfiguration <int> .Codec, GenericType <IntStreamingCodec> .Class)
.Build())
.SetPerMapConfigurations(IMRUPerMapperConfigGeneratorConfiguration.ConfigurationModule
.Set(IMRUPerMapperConfigGeneratorConfiguration.PerMapperConfigGenerator,
GenericType <NaturalSumPerMapperConfigGenerator> .Class)
.Build())
.SetUpdateFunctionCodecsConfiguration(IMRUCodecConfiguration <int> .ConfigurationModule
.Set(IMRUCodecConfiguration <int> .Codec, GenericType <IntStreamingCodec> .Class)
.Build())
.SetReduceFunctionConfiguration(IMRUReduceFunctionConfiguration <int> .ConfigurationModule
.Set(IMRUReduceFunctionConfiguration <int> .ReduceFunction,
GenericType <IntSumReduceFunction> .Class)
.Build())
.SetPartitionedDatasetConfiguration(
RandomInputDataConfiguration.ConfigurationModule.Set(
RandomInputDataConfiguration.NumberOfPartitions,
numberofMappers.ToString()).Build())
.SetResultHandlerConfiguration(
TangFactory.GetTang()
.NewConfigurationBuilder(fileSystemConfig)
.BindImplementation(GenericType <IIMRUResultHandler <int> > .Class,
GenericType <WriteResultHandler <int> > .Class)
.BindNamedParameter(typeof(ResultOutputLocation), outputFile)
.Build())
.SetJobName("NaturalSum")
.SetNumberOfMappers(numberofMappers)
.Build());
if (results != null)
{
return(results.First());
}
return(-1);
}
/// <summary>
/// </summary>
/// <returns>The number of MapFunction instances that are part of the job.</returns>
public int Run()
{
var results = _imruClient.Submit(
new IMRUJobDefinitionBuilder()
.SetConfiguration(
IMRUConfiguration <int, int, int> .ConfigurationModule
.Set(IMRUConfiguration <int, int, int> .MapFunction, GenericType <IdentityMapFunction> .Class)
.Set(IMRUConfiguration <int, int, int> .ReduceFunction,
GenericType <IntSumReduceFunction> .Class)
.Set(IMRUConfiguration <int, int, int> .UpdateFunction,
GenericType <MapperCountUpdateFunction> .Class)
.Set(IMRUConfiguration <int, int, int> .MapInputCodec,
GenericType <IntStreamingCodec> .Class)
.Set(IMRUConfiguration <int, int, int> .MapOutputCodec,
GenericType <IntStreamingCodec> .Class)
.Set(IMRUConfiguration <int, int, int> .ResultCodec, GenericType <IntStreamingCodec> .Class)
.Build())
.SetJobName("MapperCount")
.Build()
);
return(results.First());
}
