/// <summary> /// Submits the map job /// </summary> /// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam> /// <typeparam name="TMapOutput">The return type of the Map function</typeparam> /// <typeparam name="TResult">The return type of the computation.</typeparam> /// <param name="jobDefinition">Job definition given by the user</param> /// <returns>The result of the job</returns> public IEnumerable <TResult> Submit <TMapInput, TMapOutput, TResult>(IMRUJobDefinition jobDefinition) { IConfiguration overallPerMapConfig = null; try { overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray()); } catch (Exception e) { Exceptions.Throw(e, "Issues in merging PerMapCOnfigGenerator configurations", Logger); } var mergedConfig = Configurations.Merge( jobDefinition.ReduceFunctionConfiguration, jobDefinition.UpdateFunctionConfiguration, jobDefinition.UpdateFunctionCodecsConfiguration, overallPerMapConfig); var injector = TangFactory.GetTang().NewInjector(mergedConfig); ISet <IPerMapperConfigGenerator> perMapConfigGenerators = (ISet <IPerMapperConfigGenerator>)injector.GetNamedInstance(typeof(PerMapConfigGeneratorSet)); injector.BindVolatileInstance(GenericType <MapFunctions <TMapInput, TMapOutput> > .Class, MakeMapFunctions <TMapInput, TMapOutput>(jobDefinition.MapFunctionConfiguration, jobDefinition.PartitionedDatasetConfiguration, perMapConfigGenerators)); var runner = injector.GetInstance <IMRURunner <TMapInput, TMapOutput, TResult> >(); return(runner.Run()); }
/// <summary> /// Build driver configuration /// </summary> /// <typeparam name="TMapInput"></typeparam> /// <typeparam name="TMapOutput"></typeparam> /// <typeparam name="TResult"></typeparam> /// <typeparam name="TPartitionType"></typeparam> /// <param name="jobDefinition"></param> /// <param name="driverHandlerConfig"></param> /// <returns></returns> private IConfiguration DriverConfiguration <TMapInput, TMapOutput, TResult, TPartitionType>( IMRUJobDefinition jobDefinition, IConfiguration driverHandlerConfig) { string driverId = string.Format("IMRU-{0}-Driver", jobDefinition.JobName); IConfiguration overallPerMapConfig = null; var configurationSerializer = new AvroConfigurationSerializer(); try { overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray()); } catch (Exception e) { Exceptions.Throw(e, "Issues in merging PerMapCOnfigGenerator configurations", Logger); } var imruDriverConfiguration = TangFactory.GetTang().NewConfigurationBuilder(new[] { driverHandlerConfig, CreateGroupCommunicationConfiguration <TMapInput, TMapOutput, TResult, TPartitionType>(jobDefinition.NumberOfMappers + 1, driverId), jobDefinition.PartitionedDatasetConfiguration, overallPerMapConfig }) .BindNamedParameter(typeof(SerializedMapConfiguration), configurationSerializer.ToString(jobDefinition.MapFunctionConfiguration)) .BindNamedParameter(typeof(SerializedUpdateConfiguration), configurationSerializer.ToString(jobDefinition.UpdateFunctionConfiguration)) .BindNamedParameter(typeof(SerializedMapInputCodecConfiguration), configurationSerializer.ToString(jobDefinition.MapInputCodecConfiguration)) .BindNamedParameter(typeof(SerializedMapInputPipelineDataConverterConfiguration), configurationSerializer.ToString(jobDefinition.MapInputPipelineDataConverterConfiguration)) .BindNamedParameter(typeof(SerializedUpdateFunctionCodecsConfiguration), configurationSerializer.ToString(jobDefinition.UpdateFunctionCodecsConfiguration)) .BindNamedParameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration), configurationSerializer.ToString(jobDefinition.MapOutputPipelineDataConverterConfiguration)) .BindNamedParameter(typeof(SerializedReduceConfiguration), configurationSerializer.ToString(jobDefinition.ReduceFunctionConfiguration)) .BindNamedParameter(typeof(SerializedResultHandlerConfiguration), configurationSerializer.ToString(jobDefinition.ResultHandlerConfiguration)) .BindNamedParameter(typeof(MemoryPerMapper), jobDefinition.MapperMemory.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(MemoryForUpdateTask), jobDefinition.UpdateTaskMemory.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(CoresPerMapper), jobDefinition.MapTaskCores.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(CoresForUpdateTask), jobDefinition.UpdateTaskCores.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(InvokeGC), jobDefinition.InvokeGarbageCollectorAfterIteration.ToString(CultureInfo.InvariantCulture)) .Build(); return(imruDriverConfiguration); }
public IEnumerable <TResult> Submit(IMRUJobDefinition jobDefinition) { var injector = TangFactory.GetTang().NewInjector(jobDefinition.Configuration); injector.BindVolatileInstance(GenericType <MapFunctions <TMapInput, TMapOutput> > .Class, MakeMapFunctions(injector)); var runner = injector.GetInstance <IMRURunner <TMapInput, TMapOutput, TResult> >(); return(runner.Run()); }
/// <summary> /// Submits the job to reefClient /// </summary> /// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam> /// <typeparam name="TMapOutput">The return type of the Map function</typeparam> /// <typeparam name="TResult">The return type of the computation.</typeparam> /// <typeparam name="TPartitionType">Type of data partition (Generic type in IInputPartition)</typeparam> /// <param name="jobDefinition">IMRU job definition given by the user</param> /// <returns>Null as results will be later written to some directory</returns> IEnumerable <TResult> IIMRUClient.Submit <TMapInput, TMapOutput, TResult, TPartitionType>(IMRUJobDefinition jobDefinition) { string driverId = string.Format("IMRU-{0}-Driver", jobDefinition.JobName); IConfiguration overallPerMapConfig = null; try { overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray()); } catch (Exception e) { Exceptions.Throw(e, "Issues in merging PerMapCOnfigGenerator configurations", Logger); } // The driver configuration contains all the needed bindings. var imruDriverConfiguration = TangFactory.GetTang().NewConfigurationBuilder(new[] { DriverConfiguration.ConfigurationModule .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnDriverStarted, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnContextActive, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnTaskCompleted, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnEvaluatorFailed, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnContextFailed, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnTaskFailed, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.CustomTraceLevel, TraceLevel.Info.ToString()) .Build(), TangFactory.GetTang().NewConfigurationBuilder() .BindStringNamedParam <GroupCommConfigurationOptions.DriverId>(driverId) .BindStringNamedParam <GroupCommConfigurationOptions.MasterTaskId>(IMRUConstants.UpdateTaskName) .BindStringNamedParam <GroupCommConfigurationOptions.GroupName>(IMRUConstants.CommunicationGroupName) .BindIntNamedParam <GroupCommConfigurationOptions.FanOut>( IMRUConstants.TreeFanout.ToString(CultureInfo.InvariantCulture) .ToString(CultureInfo.InvariantCulture)) .BindIntNamedParam <GroupCommConfigurationOptions.NumberOfTasks>( (jobDefinition.NumberOfMappers + 1).ToString(CultureInfo.InvariantCulture)) .BindImplementation(GenericType <IGroupCommDriver> .Class, GenericType <GroupCommDriver> .Class) .Build(), jobDefinition.PartitionedDatasetConfiguration, overallPerMapConfig }) .BindNamedParameter(typeof(SerializedMapConfiguration), _configurationSerializer.ToString(jobDefinition.MapFunctionConfiguration)) .BindNamedParameter(typeof(SerializedUpdateConfiguration), _configurationSerializer.ToString(jobDefinition.UpdateFunctionConfiguration)) .BindNamedParameter(typeof(SerializedMapInputCodecConfiguration), _configurationSerializer.ToString(jobDefinition.MapInputCodecConfiguration)) .BindNamedParameter(typeof(SerializedMapInputPipelineDataConverterConfiguration), _configurationSerializer.ToString(jobDefinition.MapInputPipelineDataConverterConfiguration)) .BindNamedParameter(typeof(SerializedUpdateFunctionCodecsConfiguration), _configurationSerializer.ToString(jobDefinition.UpdateFunctionCodecsConfiguration)) .BindNamedParameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration), _configurationSerializer.ToString(jobDefinition.MapOutputPipelineDataConverterConfiguration)) .BindNamedParameter(typeof(SerializedReduceConfiguration), _configurationSerializer.ToString(jobDefinition.ReduceFunctionConfiguration)) .BindNamedParameter(typeof(SerializedResultHandlerConfiguration), _configurationSerializer.ToString(jobDefinition.ResultHandlerConfiguration)) .BindNamedParameter(typeof(MemoryPerMapper), jobDefinition.MapperMemory.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(MemoryForUpdateTask), jobDefinition.UpdateTaskMemory.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(CoresPerMapper), jobDefinition.MapTaskCores.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(CoresForUpdateTask), jobDefinition.UpdateTaskCores.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(InvokeGC), jobDefinition.InvokeGarbageCollectorAfterIteration.ToString(CultureInfo.InvariantCulture)) .Build(); // The JobSubmission contains the Driver configuration as well as the files needed on the Driver. var imruJobSubmission = _jobRequestBuilder .AddDriverConfiguration(imruDriverConfiguration) .AddGlobalAssemblyForType(typeof(IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType>)) .SetJobIdentifier(jobDefinition.JobName) .SetDriverMemory(5000) .Build(); _jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(imruJobSubmission); return(null); }