/// <summary> /// Create IMRU Job Definition with IMRU required configurations /// </summary> protected virtual IMRUJobDefinition CreateIMRUJobDefinitionBuilder(int numberofMappers, int chunkSize, int numIterations, int dim, int mapperMemory, int updateTaskMemory, int numberOfRetryInRecovery, int?numberOfChecksBeforeCancellingJob = null) { var builder = new IMRUJobDefinitionBuilder() .SetMapFunctionConfiguration(BuildMapperFunctionConfig()) .SetUpdateFunctionConfiguration(BuildUpdateFunctionConfiguration(numberofMappers, numIterations, dim)) .SetMapInputCodecConfiguration(BuildMapInputCodecConfig()) .SetUpdateFunctionCodecsConfiguration(BuildUpdateFunctionCodecsConfig()) .SetReduceFunctionConfiguration(BuildReduceFunctionConfig()) .SetMapInputPipelineDataConverterConfiguration(BuildDataConverterConfig(chunkSize)) .SetMapOutputPipelineDataConverterConfiguration(BuildDataConverterConfig(chunkSize)) .SetPartitionedDatasetConfiguration(BuildPartitionedDatasetConfiguration(numberofMappers)) .SetResultHandlerConfiguration(BuildResultHandlerConfig()) .SetJobName(IMRUJobName) .SetNumberOfMappers(numberofMappers) .SetMapperMemory(mapperMemory) .SetUpdateTaskMemory(updateTaskMemory) .SetMaxRetryNumberInRecovery(numberOfRetryInRecovery); if (numberOfChecksBeforeCancellingJob.HasValue) { var cancelConfig = TangFactory.GetTang().NewConfigurationBuilder() .BindImplementation(GenericType <IJobCancelledDetector> .Class, GenericType <JobCancellationDetectoBasedOnCheckCount> .Class) .BindNamedParameter(typeof(JobCancellationDetectoBasedOnCheckCount.NumberOfChecksBeforeCancelling), numberOfChecksBeforeCancellingJob.Value.ToString()) .BindNamedParameter(typeof(SleepIntervalParameter), "1") .Build(); builder.SetJobCancellationConfiguration(cancelConfig); } return(builder.Build()); }
/// <summary> /// This generates empty driver configuration which can be used to construct instance of the IMRUDriver, /// but is not functional. /// this is used to unit test specific code path (like JobCancelledEvent in this case) /// </summary> private IConfiguration GetDriverConfig <TMapInput, TMapOutput, TResult, TPartitionType>() { var testConfig = TangFactory.GetTang().NewConfigurationBuilder() .BindImplementation(GenericType <IPartitionedInputDataSet> .Class, GenericType <TestPartitionedInputDataSet> .Class) .BindImplementation(GenericType <IEvaluatorRequestor> .Class, GenericType <TestEvaluatorRequestor> .Class) .Build(); var jobDefinition = new IMRUJobDefinitionBuilder() .SetJobName("Test") .SetMapFunctionConfiguration(testConfig) .SetMapInputCodecConfiguration(testConfig) .SetUpdateFunctionCodecsConfiguration(testConfig) .SetReduceFunctionConfiguration(testConfig) .SetUpdateFunctionConfiguration(testConfig) .SetPartitionedDatasetConfiguration(testConfig) .Build(); var _configurationSerializer = new AvroConfigurationSerializer(); var overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray()); var driverConfig = TangFactory.GetTang().NewConfigurationBuilder(new[] { DriverConfiguration.ConfigurationModule .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnDriverStarted, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnContextActive, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnTaskCompleted, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnEvaluatorFailed, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnContextFailed, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnTaskFailed, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.OnTaskRunning, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class) .Set(DriverConfiguration.CustomTraceLevel, TraceLevel.Info.ToString()) .Build(), TangFactory.GetTang().NewConfigurationBuilder() .BindStringNamedParam <GroupCommConfigurationOptions.DriverId>("driverId") .BindStringNamedParam <GroupCommConfigurationOptions.MasterTaskId>(IMRUConstants.UpdateTaskName) .BindStringNamedParam <GroupCommConfigurationOptions.GroupName>( IMRUConstants.CommunicationGroupName) .BindIntNamedParam <GroupCommConfigurationOptions.FanOut>( IMRUConstants.TreeFanout.ToString(CultureInfo.InvariantCulture) .ToString(CultureInfo.InvariantCulture)) .BindIntNamedParam <GroupCommConfigurationOptions.NumberOfTasks>( (jobDefinition.NumberOfMappers + 1).ToString(CultureInfo.InvariantCulture)) .BindImplementation(GenericType <IGroupCommDriver> .Class, GenericType <GroupCommDriver> .Class) .Build(), jobDefinition.PartitionedDatasetConfiguration, overallPerMapConfig, jobDefinition.JobCancelSignalConfiguration }) .BindNamedParameter(typeof(SerializedUpdateTaskStateConfiguration), _configurationSerializer.ToString(jobDefinition.UpdateTaskStateConfiguration)) .BindNamedParameter(typeof(SerializedMapTaskStateConfiguration), _configurationSerializer.ToString(jobDefinition.MapTaskStateConfiguration)) .BindNamedParameter(typeof(SerializedMapConfiguration), _configurationSerializer.ToString(jobDefinition.MapFunctionConfiguration)) .BindNamedParameter(typeof(SerializedUpdateConfiguration), _configurationSerializer.ToString(jobDefinition.UpdateFunctionConfiguration)) .BindNamedParameter(typeof(SerializedMapInputCodecConfiguration), _configurationSerializer.ToString(jobDefinition.MapInputCodecConfiguration)) .BindNamedParameter(typeof(SerializedMapInputPipelineDataConverterConfiguration), _configurationSerializer.ToString(jobDefinition.MapInputPipelineDataConverterConfiguration)) .BindNamedParameter(typeof(SerializedUpdateFunctionCodecsConfiguration), _configurationSerializer.ToString(jobDefinition.UpdateFunctionCodecsConfiguration)) .BindNamedParameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration), _configurationSerializer.ToString(jobDefinition.MapOutputPipelineDataConverterConfiguration)) .BindNamedParameter(typeof(SerializedReduceConfiguration), _configurationSerializer.ToString(jobDefinition.ReduceFunctionConfiguration)) .BindNamedParameter(typeof(SerializedResultHandlerConfiguration), _configurationSerializer.ToString(jobDefinition.ResultHandlerConfiguration)) .BindNamedParameter(typeof(MemoryPerMapper), jobDefinition.MapperMemory.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(MemoryForUpdateTask), jobDefinition.UpdateTaskMemory.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(CoresPerMapper), jobDefinition.MapTaskCores.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(CoresForUpdateTask), jobDefinition.UpdateTaskCores.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(MaxRetryNumberInRecovery), jobDefinition.MaxRetryNumberInRecovery.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(InvokeGC), jobDefinition.InvokeGarbageCollectorAfterIteration.ToString(CultureInfo.InvariantCulture)) .Build(); return(driverConfig); }