/// <summary>
/// Submits the map job
/// </summary>
/// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam>
/// <typeparam name="TMapOutput">The return type of the Map function</typeparam>
/// <typeparam name="TResult">The return type of the computation.</typeparam>
/// <param name="jobDefinition">Job definition given by the user</param>
/// <returns>The result of the job</returns>
public IEnumerable <TResult> Submit <TMapInput, TMapOutput, TResult>(IMRUJobDefinition jobDefinition)
{
IConfiguration overallPerMapConfig = null;
try
{
overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray());
}
catch (Exception e)
{
Exceptions.Throw(e, "Issues in merging PerMapCOnfigGenerator configurations", Logger);
}
var mergedConfig = Configurations.Merge(
jobDefinition.ReduceFunctionConfiguration,
jobDefinition.UpdateFunctionConfiguration,
jobDefinition.UpdateFunctionCodecsConfiguration,
overallPerMapConfig);
var injector = TangFactory.GetTang().NewInjector(mergedConfig);
ISet <IPerMapperConfigGenerator> perMapConfigGenerators =
(ISet <IPerMapperConfigGenerator>)injector.GetNamedInstance(typeof(PerMapConfigGeneratorSet));
injector.BindVolatileInstance(GenericType <MapFunctions <TMapInput, TMapOutput> > .Class,
MakeMapFunctions <TMapInput, TMapOutput>(jobDefinition.MapFunctionConfiguration, jobDefinition.PartitionedDatasetConfiguration, perMapConfigGenerators));
var runner = injector.GetInstance <IMRURunner <TMapInput, TMapOutput, TResult> >();
return(runner.Run());
}
/// <summary>
/// Build driver configuration
/// </summary>
/// <typeparam name="TMapInput"></typeparam>
/// <typeparam name="TMapOutput"></typeparam>
/// <typeparam name="TResult"></typeparam>
/// <typeparam name="TPartitionType"></typeparam>
/// <param name="jobDefinition"></param>
/// <param name="driverHandlerConfig"></param>
/// <returns></returns>
private IConfiguration DriverConfiguration <TMapInput, TMapOutput, TResult, TPartitionType>(
IMRUJobDefinition jobDefinition,
IConfiguration driverHandlerConfig)
{
string driverId = string.Format("IMRU-{0}-Driver", jobDefinition.JobName);
IConfiguration overallPerMapConfig = null;
var configurationSerializer = new AvroConfigurationSerializer();
try
{
overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray());
}
catch (Exception e)
{
Exceptions.Throw(e, "Issues in merging PerMapCOnfigGenerator configurations", Logger);
}
var imruDriverConfiguration = TangFactory.GetTang().NewConfigurationBuilder(new[]
{
driverHandlerConfig,
CreateGroupCommunicationConfiguration <TMapInput, TMapOutput, TResult, TPartitionType>(jobDefinition.NumberOfMappers + 1,
driverId),
jobDefinition.PartitionedDatasetConfiguration,
overallPerMapConfig
})
.BindNamedParameter(typeof(SerializedMapConfiguration),
configurationSerializer.ToString(jobDefinition.MapFunctionConfiguration))
.BindNamedParameter(typeof(SerializedUpdateConfiguration),
configurationSerializer.ToString(jobDefinition.UpdateFunctionConfiguration))
.BindNamedParameter(typeof(SerializedMapInputCodecConfiguration),
configurationSerializer.ToString(jobDefinition.MapInputCodecConfiguration))
.BindNamedParameter(typeof(SerializedMapInputPipelineDataConverterConfiguration),
configurationSerializer.ToString(jobDefinition.MapInputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedUpdateFunctionCodecsConfiguration),
configurationSerializer.ToString(jobDefinition.UpdateFunctionCodecsConfiguration))
.BindNamedParameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration),
configurationSerializer.ToString(jobDefinition.MapOutputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedReduceConfiguration),
configurationSerializer.ToString(jobDefinition.ReduceFunctionConfiguration))
.BindNamedParameter(typeof(SerializedResultHandlerConfiguration),
configurationSerializer.ToString(jobDefinition.ResultHandlerConfiguration))
.BindNamedParameter(typeof(MemoryPerMapper),
jobDefinition.MapperMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(MemoryForUpdateTask),
jobDefinition.UpdateTaskMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresPerMapper),
jobDefinition.MapTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresForUpdateTask),
jobDefinition.UpdateTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(InvokeGC),
jobDefinition.InvokeGarbageCollectorAfterIteration.ToString(CultureInfo.InvariantCulture))
.Build();
return(imruDriverConfiguration);
}
public IEnumerable <TResult> Submit(IMRUJobDefinition jobDefinition)
{
var injector = TangFactory.GetTang().NewInjector(jobDefinition.Configuration);
injector.BindVolatileInstance(GenericType <MapFunctions <TMapInput, TMapOutput> > .Class,
MakeMapFunctions(injector));
var runner = injector.GetInstance <IMRURunner <TMapInput, TMapOutput, TResult> >();
return(runner.Run());
}
/// <summary>
/// Submits the job to reefClient
/// </summary>
/// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam>
/// <typeparam name="TMapOutput">The return type of the Map function</typeparam>
/// <typeparam name="TResult">The return type of the computation.</typeparam>
/// <typeparam name="TPartitionType">Type of data partition (Generic type in IInputPartition)</typeparam>
/// <param name="jobDefinition">IMRU job definition given by the user</param>
/// <returns>Null as results will be later written to some directory</returns>
IEnumerable <TResult> IIMRUClient.Submit <TMapInput, TMapOutput, TResult, TPartitionType>(IMRUJobDefinition jobDefinition)
{
string driverId = string.Format("IMRU-{0}-Driver", jobDefinition.JobName);
IConfiguration overallPerMapConfig = null;
try
{
overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray());
}
catch (Exception e)
{
Exceptions.Throw(e, "Issues in merging PerMapCOnfigGenerator configurations", Logger);
}
// The driver configuration contains all the needed bindings.
var imruDriverConfiguration = TangFactory.GetTang().NewConfigurationBuilder(new[]
{
DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnDriverStarted,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnContextActive,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnTaskCompleted,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnEvaluatorFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnContextFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnTaskFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.CustomTraceLevel, TraceLevel.Info.ToString())
.Build(),
TangFactory.GetTang().NewConfigurationBuilder()
.BindStringNamedParam <GroupCommConfigurationOptions.DriverId>(driverId)
.BindStringNamedParam <GroupCommConfigurationOptions.MasterTaskId>(IMRUConstants.UpdateTaskName)
.BindStringNamedParam <GroupCommConfigurationOptions.GroupName>(IMRUConstants.CommunicationGroupName)
.BindIntNamedParam <GroupCommConfigurationOptions.FanOut>(
IMRUConstants.TreeFanout.ToString(CultureInfo.InvariantCulture)
.ToString(CultureInfo.InvariantCulture))
.BindIntNamedParam <GroupCommConfigurationOptions.NumberOfTasks>(
(jobDefinition.NumberOfMappers + 1).ToString(CultureInfo.InvariantCulture))
.BindImplementation(GenericType <IGroupCommDriver> .Class, GenericType <GroupCommDriver> .Class)
.Build(),
jobDefinition.PartitionedDatasetConfiguration,
overallPerMapConfig
})
.BindNamedParameter(typeof(SerializedMapConfiguration),
_configurationSerializer.ToString(jobDefinition.MapFunctionConfiguration))
.BindNamedParameter(typeof(SerializedUpdateConfiguration),
_configurationSerializer.ToString(jobDefinition.UpdateFunctionConfiguration))
.BindNamedParameter(typeof(SerializedMapInputCodecConfiguration),
_configurationSerializer.ToString(jobDefinition.MapInputCodecConfiguration))
.BindNamedParameter(typeof(SerializedMapInputPipelineDataConverterConfiguration),
_configurationSerializer.ToString(jobDefinition.MapInputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedUpdateFunctionCodecsConfiguration),
_configurationSerializer.ToString(jobDefinition.UpdateFunctionCodecsConfiguration))
.BindNamedParameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration),
_configurationSerializer.ToString(jobDefinition.MapOutputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedReduceConfiguration),
_configurationSerializer.ToString(jobDefinition.ReduceFunctionConfiguration))
.BindNamedParameter(typeof(SerializedResultHandlerConfiguration),
_configurationSerializer.ToString(jobDefinition.ResultHandlerConfiguration))
.BindNamedParameter(typeof(MemoryPerMapper),
jobDefinition.MapperMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(MemoryForUpdateTask),
jobDefinition.UpdateTaskMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresPerMapper),
jobDefinition.MapTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresForUpdateTask),
jobDefinition.UpdateTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(InvokeGC),
jobDefinition.InvokeGarbageCollectorAfterIteration.ToString(CultureInfo.InvariantCulture))
.Build();
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
var imruJobSubmission = _jobRequestBuilder
.AddDriverConfiguration(imruDriverConfiguration)
.AddGlobalAssemblyForType(typeof(IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType>))
.SetJobIdentifier(jobDefinition.JobName)
.SetDriverMemory(5000)
.Build();
_jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(imruJobSubmission);
return(null);
}
