/// <summary>
/// Runs DriverRestart using the IREEFClient passed into the constructor.
/// </summary>
private void Run()
{
// The driver configuration contains all the needed bindings.
var driverConfiguration = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnDriverStarted, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnDriverRestarted, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnContextActive, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnTaskRunning, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnTaskFailed, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnTaskCompleted, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnDriverRestartCompleted, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnDriverRestartContextActive, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnDriverRestartTaskRunning, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnEvaluatorFailed, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnDriverRestartEvaluatorFailed, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnDriverReconnect, GenericType <DefaultYarnClusterHttpDriverConnection> .Class)
.Set(DriverConfiguration.DriverRestartEvaluatorRecoverySeconds, (5 * 60).ToString())
.Set(DriverConfiguration.MaxApplicationSubmissions, 2.ToString())
.Build();
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
var restartJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder()
.AddDriverConfiguration(driverConfiguration)
.AddGlobalAssemblyForType(typeof(HelloRestartDriver))
.SetJobIdentifier("DriverRestart_" + Guid.NewGuid().ToString().Substring(0, 6))
.Build();
_reefClient.SubmitAndGetJobStatus(restartJobSubmission);
}
/// <summary>
/// Runs HelloREEF using the IREEFClient passed into the constructor.
/// </summary>
private void Run()
{
// The driver configuration contains all the needed bindings.
var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloDriver> .Class)
.Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriver> .Class)
.Build();
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
var helloJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder()
.AddDriverConfiguration(helloDriverConfiguration)
.AddGlobalAssemblyForType(typeof(HelloDriver))
.SetJobIdentifier("HelloREEF")
.Build();
_reefClient.Submit(helloJobSubmission);
}
private IJobSubmissionResult Run()
{
var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloAllocatedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <AnotherHelloAllocatedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnContextActive, GenericType <HelloActiveContextHandler> .Class)
.Set(DriverConfiguration.OnTaskMessage, GenericType <HelloTaskMessageHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorFailed, GenericType <HelloFailedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnTaskFailed, GenericType <HelloFailedTaskHandler> .Class)
.Set(DriverConfiguration.OnTaskRunning, GenericType <HelloRunningTaskHandler> .Class)
.Set(DriverConfiguration.OnTaskCompleted, GenericType <HelloTaskCompletedHandler> .Class)
.Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriverStartHandler> .Class)
.Set(DriverConfiguration.OnHttpEvent, GenericType <HelloHttpHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorCompleted, GenericType <HelloCompletedEvaluatorHandler> .Class)
.Set(DriverConfiguration.CustomTraceListeners, GenericType <DefaultCustomTraceListener> .Class)
.Set(DriverConfiguration.CustomTraceLevel, Level.Info.ToString())
.Set(DriverConfiguration.OnDriverRestarted, GenericType <HelloRestartHandler> .Class)
.Set(DriverConfiguration.OnDriverReconnect, GenericType <DefaultLocalHttpDriverConnection> .Class)
.Set(DriverConfiguration.OnDriverRestartContextActive, GenericType <HelloDriverRestartActiveContextHandler> .Class)
.Set(DriverConfiguration.OnDriverRestartTaskRunning, GenericType <HelloDriverRestartRunningTaskHandler> .Class)
.Build();
var driverConfig = TangFactory.GetTang().NewConfigurationBuilder(helloDriverConfiguration)
.BindSetEntry <DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(HelloTask).Assembly.GetName().Name)
.BindSetEntry <DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(NameClient).Assembly.GetName().Name)
.Build();
var helloJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder()
.AddDriverConfiguration(driverConfig)
.AddGlobalAssemblyForType(typeof(HelloDriverStartHandler))
.SetJobIdentifier("HelloDriver")
.Build();
IJobSubmissionResult jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(helloJobSubmission);
return(jobSubmissionResult);
}
/// <summary>
/// Submits the job to reefClient
/// </summary>
/// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam>
/// <typeparam name="TMapOutput">The return type of the Map function</typeparam>
/// <typeparam name="TResult">The return type of the computation.</typeparam>
/// <param name="jobDefinition">IMRU job definition given by the user</param>
/// <returns>Null as results will be later written to some directory</returns>
IEnumerable <TResult> IIMRUClient.Submit <TMapInput, TMapOutput, TResult>(IMRUJobDefinition jobDefinition)
{
string driverId = string.Format("IMRU-{0}-Driver", jobDefinition.JobName);
IConfiguration overallPerMapConfig = null;
try
{
overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray());
}
catch (Exception e)
{
Exceptions.Throw(e, "Issues in merging PerMapCOnfigGenerator configurations", Logger);
}
// The driver configuration contains all the needed bindings.
var imruDriverConfiguration = TangFactory.GetTang().NewConfigurationBuilder(new[]
{
DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult> > .Class)
.Set(DriverConfiguration.OnDriverStarted,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult> > .Class)
.Set(DriverConfiguration.OnContextActive,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult> > .Class)
.Set(DriverConfiguration.OnTaskCompleted,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult> > .Class)
.Set(DriverConfiguration.OnEvaluatorFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult> > .Class)
.Set(DriverConfiguration.CustomTraceLevel, TraceLevel.Info.ToString())
.Build(),
TangFactory.GetTang().NewConfigurationBuilder()
.BindStringNamedParam <GroupCommConfigurationOptions.DriverId>(driverId)
.BindStringNamedParam <GroupCommConfigurationOptions.MasterTaskId>(IMRUConstants.UpdateTaskName)
.BindStringNamedParam <GroupCommConfigurationOptions.GroupName>(IMRUConstants.CommunicationGroupName)
.BindIntNamedParam <GroupCommConfigurationOptions.FanOut>(
IMRUConstants.TreeFanout.ToString(CultureInfo.InvariantCulture)
.ToString(CultureInfo.InvariantCulture))
.BindIntNamedParam <GroupCommConfigurationOptions.NumberOfTasks>(
(jobDefinition.NumberOfMappers + 1).ToString(CultureInfo.InvariantCulture))
.BindImplementation(GenericType <IGroupCommDriver> .Class, GenericType <GroupCommDriver> .Class)
.Build(),
jobDefinition.PartitionedDatasetConfiguration,
overallPerMapConfig
})
.BindNamedParameter(typeof(SerializedMapConfiguration),
_configurationSerializer.ToString(jobDefinition.MapFunctionConfiguration))
.BindNamedParameter(typeof(SerializedUpdateConfiguration),
_configurationSerializer.ToString(jobDefinition.UpdateFunctionConfiguration))
.BindNamedParameter(typeof(SerializedMapInputCodecConfiguration),
_configurationSerializer.ToString(jobDefinition.MapInputCodecConfiguration))
.BindNamedParameter(typeof(SerializedMapInputPipelineDataConverterConfiguration),
_configurationSerializer.ToString(jobDefinition.MapInputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedUpdateFunctionCodecsConfiguration),
_configurationSerializer.ToString(jobDefinition.UpdateFunctionCodecsConfiguration))
.BindNamedParameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration),
_configurationSerializer.ToString(jobDefinition.MapOutputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedReduceConfiguration),
_configurationSerializer.ToString(jobDefinition.ReduceFunctionConfiguration))
.BindNamedParameter(typeof(SerializedResultHandlerConfiguration),
_configurationSerializer.ToString(jobDefinition.ResultHandlerConfiguration))
.BindNamedParameter(typeof(MemoryPerMapper),
jobDefinition.MapperMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(MemoryForUpdateTask),
jobDefinition.UpdateTaskMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresPerMapper),
jobDefinition.MapTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresForUpdateTask),
jobDefinition.UpdateTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(InvokeGC),
jobDefinition.InvokeGarbageCollectorAfterIteration.ToString(CultureInfo.InvariantCulture))
.Build();
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
var imruJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder()
.AddDriverConfiguration(imruDriverConfiguration)
.AddGlobalAssemblyForType(typeof(IMRUDriver <TMapInput, TMapOutput, TResult>))
.SetJobIdentifier(jobDefinition.JobName)
.Build();
_jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(imruJobSubmission);
return(null);
}
