/// <summary>
/// Create IMRU Job Definition with IMRU required configurations
/// </summary>
protected virtual IMRUJobDefinition CreateIMRUJobDefinitionBuilder(int numberofMappers,
int chunkSize,
int numIterations,
int dim,
int mapperMemory,
int updateTaskMemory,
int numberOfRetryInRecovery,
int?numberOfChecksBeforeCancellingJob = null)
{
var builder = new IMRUJobDefinitionBuilder()
.SetMapFunctionConfiguration(BuildMapperFunctionConfig())
.SetUpdateFunctionConfiguration(BuildUpdateFunctionConfiguration(numberofMappers, numIterations, dim))
.SetMapInputCodecConfiguration(BuildMapInputCodecConfig())
.SetUpdateFunctionCodecsConfiguration(BuildUpdateFunctionCodecsConfig())
.SetReduceFunctionConfiguration(BuildReduceFunctionConfig())
.SetMapInputPipelineDataConverterConfiguration(BuildDataConverterConfig(chunkSize))
.SetMapOutputPipelineDataConverterConfiguration(BuildDataConverterConfig(chunkSize))
.SetPartitionedDatasetConfiguration(BuildPartitionedDatasetConfiguration(numberofMappers))
.SetResultHandlerConfiguration(BuildResultHandlerConfig())
.SetJobName(IMRUJobName)
.SetNumberOfMappers(numberofMappers)
.SetMapperMemory(mapperMemory)
.SetUpdateTaskMemory(updateTaskMemory)
.SetMaxRetryNumberInRecovery(numberOfRetryInRecovery);
if (numberOfChecksBeforeCancellingJob.HasValue)
{
var cancelConfig = TangFactory.GetTang().NewConfigurationBuilder()
.BindImplementation(GenericType <IJobCancelledDetector> .Class, GenericType <JobCancellationDetectoBasedOnCheckCount> .Class)
.BindNamedParameter(typeof(JobCancellationDetectoBasedOnCheckCount.NumberOfChecksBeforeCancelling), numberOfChecksBeforeCancellingJob.Value.ToString())
.BindNamedParameter(typeof(SleepIntervalParameter), "1")
.Build();
builder.SetJobCancellationConfiguration(cancelConfig);
}
return(builder.Build());
}
/// <summary>
/// This generates empty driver configuration which can be used to construct instance of the IMRUDriver,
/// but is not functional.
/// this is used to unit test specific code path (like JobCancelledEvent in this case)
/// </summary>
private IConfiguration GetDriverConfig <TMapInput, TMapOutput, TResult, TPartitionType>()
{
var testConfig = TangFactory.GetTang().NewConfigurationBuilder()
.BindImplementation(GenericType <IPartitionedInputDataSet> .Class, GenericType <TestPartitionedInputDataSet> .Class)
.BindImplementation(GenericType <IEvaluatorRequestor> .Class, GenericType <TestEvaluatorRequestor> .Class)
.Build();
var jobDefinition = new IMRUJobDefinitionBuilder()
.SetJobName("Test")
.SetMapFunctionConfiguration(testConfig)
.SetMapInputCodecConfiguration(testConfig)
.SetUpdateFunctionCodecsConfiguration(testConfig)
.SetReduceFunctionConfiguration(testConfig)
.SetUpdateFunctionConfiguration(testConfig)
.SetPartitionedDatasetConfiguration(testConfig)
.Build();
var _configurationSerializer = new AvroConfigurationSerializer();
var overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray());
var driverConfig = TangFactory.GetTang().NewConfigurationBuilder(new[]
{
DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnDriverStarted,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnContextActive,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnTaskCompleted,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnEvaluatorFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnContextFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnTaskFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnTaskRunning,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.CustomTraceLevel, TraceLevel.Info.ToString())
.Build(),
TangFactory.GetTang().NewConfigurationBuilder()
.BindStringNamedParam <GroupCommConfigurationOptions.DriverId>("driverId")
.BindStringNamedParam <GroupCommConfigurationOptions.MasterTaskId>(IMRUConstants.UpdateTaskName)
.BindStringNamedParam <GroupCommConfigurationOptions.GroupName>(
IMRUConstants.CommunicationGroupName)
.BindIntNamedParam <GroupCommConfigurationOptions.FanOut>(
IMRUConstants.TreeFanout.ToString(CultureInfo.InvariantCulture)
.ToString(CultureInfo.InvariantCulture))
.BindIntNamedParam <GroupCommConfigurationOptions.NumberOfTasks>(
(jobDefinition.NumberOfMappers + 1).ToString(CultureInfo.InvariantCulture))
.BindImplementation(GenericType <IGroupCommDriver> .Class, GenericType <GroupCommDriver> .Class)
.Build(),
jobDefinition.PartitionedDatasetConfiguration,
overallPerMapConfig,
jobDefinition.JobCancelSignalConfiguration
})
.BindNamedParameter(typeof(SerializedUpdateTaskStateConfiguration),
_configurationSerializer.ToString(jobDefinition.UpdateTaskStateConfiguration))
.BindNamedParameter(typeof(SerializedMapTaskStateConfiguration),
_configurationSerializer.ToString(jobDefinition.MapTaskStateConfiguration))
.BindNamedParameter(typeof(SerializedMapConfiguration),
_configurationSerializer.ToString(jobDefinition.MapFunctionConfiguration))
.BindNamedParameter(typeof(SerializedUpdateConfiguration),
_configurationSerializer.ToString(jobDefinition.UpdateFunctionConfiguration))
.BindNamedParameter(typeof(SerializedMapInputCodecConfiguration),
_configurationSerializer.ToString(jobDefinition.MapInputCodecConfiguration))
.BindNamedParameter(typeof(SerializedMapInputPipelineDataConverterConfiguration),
_configurationSerializer.ToString(jobDefinition.MapInputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedUpdateFunctionCodecsConfiguration),
_configurationSerializer.ToString(jobDefinition.UpdateFunctionCodecsConfiguration))
.BindNamedParameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration),
_configurationSerializer.ToString(jobDefinition.MapOutputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedReduceConfiguration),
_configurationSerializer.ToString(jobDefinition.ReduceFunctionConfiguration))
.BindNamedParameter(typeof(SerializedResultHandlerConfiguration),
_configurationSerializer.ToString(jobDefinition.ResultHandlerConfiguration))
.BindNamedParameter(typeof(MemoryPerMapper),
jobDefinition.MapperMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(MemoryForUpdateTask),
jobDefinition.UpdateTaskMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresPerMapper),
jobDefinition.MapTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresForUpdateTask),
jobDefinition.UpdateTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(MaxRetryNumberInRecovery),
jobDefinition.MaxRetryNumberInRecovery.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(InvokeGC),
jobDefinition.InvokeGarbageCollectorAfterIteration.ToString(CultureInfo.InvariantCulture))
.Build();
return(driverConfig);
}
