/// <summary> /// Runs the actual broadcast and reduce job /// </summary> public void Run(int numberofMappers, int chunkSize, int numIterations, int dim, int mapperMemory, int updateTaskMemory) { var updateFunctionConfig = TangFactory.GetTang().NewConfigurationBuilder(IMRUUpdateConfiguration <int[], int[], int[]> .ConfigurationModule .Set(IMRUUpdateConfiguration <int[], int[], int[]> .UpdateFunction, GenericType <BroadcastSenderReduceReceiverUpdateFunction> .Class).Build()) .BindNamedParameter(typeof(BroadcastReduceConfiguration.NumberOfIterations), numIterations.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(BroadcastReduceConfiguration.Dimensions), dim.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(BroadcastReduceConfiguration.NumWorkers), numberofMappers.ToString(CultureInfo.InvariantCulture)) .Build(); var dataConverterConfig1 = TangFactory.GetTang() .NewConfigurationBuilder(IMRUPipelineDataConverterConfiguration <int[]> .ConfigurationModule .Set(IMRUPipelineDataConverterConfiguration <int[]> .MapInputPiplelineDataConverter, GenericType <PipelineIntDataConverter> .Class).Build()) .BindNamedParameter(typeof(BroadcastReduceConfiguration.ChunkSize), chunkSize.ToString(CultureInfo.InvariantCulture)) .Build(); var dataConverterConfig2 = TangFactory.GetTang() .NewConfigurationBuilder(IMRUPipelineDataConverterConfiguration <int[]> .ConfigurationModule .Set(IMRUPipelineDataConverterConfiguration <int[]> .MapInputPiplelineDataConverter, GenericType <PipelineIntDataConverter> .Class).Build()) .BindNamedParameter(typeof(BroadcastReduceConfiguration.ChunkSize), chunkSize.ToString(CultureInfo.InvariantCulture)) .Build(); var results = _imruClient.Submit <int[], int[], int[]>( new IMRUJobDefinitionBuilder() .SetMapFunctionConfiguration(IMRUMapConfiguration <int[], int[]> .ConfigurationModule .Set(IMRUMapConfiguration <int[], int[]> .MapFunction, GenericType <BroadcastReceiverReduceSenderMapFunction> .Class) .Build()) .SetUpdateFunctionConfiguration(updateFunctionConfig) .SetMapInputCodecConfiguration(IMRUCodecConfiguration <int[]> .ConfigurationModule .Set(IMRUCodecConfiguration <int[]> .Codec, GenericType <IntArrayStreamingCodec> .Class) .Build()) .SetUpdateFunctionCodecsConfiguration(IMRUCodecConfiguration <int[]> .ConfigurationModule .Set(IMRUCodecConfiguration <int[]> .Codec, GenericType <IntArrayStreamingCodec> .Class) .Build()) .SetReduceFunctionConfiguration(IMRUReduceFunctionConfiguration <int[]> .ConfigurationModule .Set(IMRUReduceFunctionConfiguration <int[]> .ReduceFunction, GenericType <IntArraySumReduceFunction> .Class) .Build()) .SetMapInputPipelineDataConverterConfiguration(dataConverterConfig1) .SetMapOutputPipelineDataConverterConfiguration(dataConverterConfig2) .SetPartitionedDatasetConfiguration( RandomInputDataConfiguration.ConfigurationModule.Set(RandomInputDataConfiguration.NumberOfPartitions, numberofMappers.ToString()).Build()) .SetJobName("BroadcastReduce") .SetNumberOfMappers(numberofMappers) .SetMapperMemory(mapperMemory) .SetUpdateTaskMemory(updateTaskMemory) .Build()); }
/// <summary> /// Runs the actual broadcast and reduce job /// </summary> public void Run(int numberofMappers, int chunkSize, int numIterations, int dim, int mapperMemory, int updateTaskMemory) { var results = _imruClient.Submit <int[], int[], int[], Stream>( CreateJobDefinitionBuilder(numberofMappers, chunkSize, numIterations, dim, mapperMemory, updateTaskMemory) .SetMapFunctionConfiguration(BuildMapperFunctionConfig()) .Build()); }
/// <summary> /// Runs the actual natural sum job. /// </summary> /// <returns>The result of the natural sum IMRU job.</returns> public int Run(int numberofMappers, string outputFile, IConfiguration fileSystemConfig) { var results = _imruClient.Submit <int, int, int, Stream>( new IMRUJobDefinitionBuilder() .SetMapFunctionConfiguration(IMRUMapConfiguration <int, int> .ConfigurationModule .Set(IMRUMapConfiguration <int, int> .MapFunction, GenericType <NaturalSumMapFunction> .Class) .Build()) .SetUpdateFunctionConfiguration( IMRUUpdateConfiguration <int, int, int> .ConfigurationModule .Set(IMRUUpdateConfiguration <int, int, int> .UpdateFunction, GenericType <SingleIterUpdateFunction> .Class) .Build()) .SetMapInputCodecConfiguration(IMRUCodecConfiguration <int> .ConfigurationModule .Set(IMRUCodecConfiguration <int> .Codec, GenericType <IntStreamingCodec> .Class) .Build()) .SetPerMapConfigurations(IMRUPerMapperConfigGeneratorConfiguration.ConfigurationModule .Set(IMRUPerMapperConfigGeneratorConfiguration.PerMapperConfigGenerator, GenericType <NaturalSumPerMapperConfigGenerator> .Class) .Build()) .SetUpdateFunctionCodecsConfiguration(IMRUCodecConfiguration <int> .ConfigurationModule .Set(IMRUCodecConfiguration <int> .Codec, GenericType <IntStreamingCodec> .Class) .Build()) .SetReduceFunctionConfiguration(IMRUReduceFunctionConfiguration <int> .ConfigurationModule .Set(IMRUReduceFunctionConfiguration <int> .ReduceFunction, GenericType <IntSumReduceFunction> .Class) .Build()) .SetPartitionedDatasetConfiguration( RandomInputDataConfiguration.ConfigurationModule.Set( RandomInputDataConfiguration.NumberOfPartitions, numberofMappers.ToString()).Build()) .SetResultHandlerConfiguration( TangFactory.GetTang() .NewConfigurationBuilder(fileSystemConfig) .BindImplementation(GenericType <IIMRUResultHandler <int> > .Class, GenericType <WriteResultHandler <int> > .Class) .BindNamedParameter(typeof(ResultOutputLocation), outputFile) .Build()) .SetJobName("NaturalSum") .SetNumberOfMappers(numberofMappers) .Build()); if (results != null) { return(results.First()); } return(-1); }
/// <summary> /// </summary> /// <returns>The number of MapFunction instances that are part of the job.</returns> public int Run() { var results = _imruClient.Submit( new IMRUJobDefinitionBuilder() .SetConfiguration( IMRUConfiguration <int, int, int> .ConfigurationModule .Set(IMRUConfiguration <int, int, int> .MapFunction, GenericType <IdentityMapFunction> .Class) .Set(IMRUConfiguration <int, int, int> .ReduceFunction, GenericType <IntSumReduceFunction> .Class) .Set(IMRUConfiguration <int, int, int> .UpdateFunction, GenericType <MapperCountUpdateFunction> .Class) .Set(IMRUConfiguration <int, int, int> .MapInputCodec, GenericType <IntStreamingCodec> .Class) .Set(IMRUConfiguration <int, int, int> .MapOutputCodec, GenericType <IntStreamingCodec> .Class) .Set(IMRUConfiguration <int, int, int> .ResultCodec, GenericType <IntStreamingCodec> .Class) .Build()) .SetJobName("MapperCount") .Build() ); return(results.First()); }