private IJobSubmissionResult Run()
{
var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloAllocatedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <AnotherHelloAllocatedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnContextActive, GenericType <HelloActiveContextHandler> .Class)
.Set(DriverConfiguration.OnTaskMessage, GenericType <HelloTaskMessageHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorFailed, GenericType <HelloFailedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnTaskFailed, GenericType <HelloFailedTaskHandler> .Class)
.Set(DriverConfiguration.OnTaskRunning, GenericType <HelloRunningTaskHandler> .Class)
.Set(DriverConfiguration.OnTaskCompleted, GenericType <HelloTaskCompletedHandler> .Class)
.Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriverStartHandler> .Class)
.Set(DriverConfiguration.OnHttpEvent, GenericType <HelloHttpHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorCompleted, GenericType <HelloCompletedEvaluatorHandler> .Class)
.Set(DriverConfiguration.CustomTraceListeners, GenericType <DefaultCustomTraceListener> .Class)
.Set(DriverConfiguration.CustomTraceLevel, Level.Info.ToString())
.Set(DriverConfiguration.OnDriverRestarted, GenericType <HelloRestartHandler> .Class)
.Set(DriverConfiguration.DriverReconnectionConfigurationProvider,
GenericType <LocalHttpDriverReconnConfigProvider> .Class)
.Set(DriverConfiguration.OnDriverRestartContextActive, GenericType <HelloDriverRestartActiveContextHandler> .Class)
.Set(DriverConfiguration.OnDriverRestartTaskRunning, GenericType <HelloDriverRestartRunningTaskHandler> .Class)
.Build();
var helloJobSubmission = _jobRequestBuilder
.AddDriverConfiguration(helloDriverConfiguration)
.AddGlobalAssemblyForType(typeof(HelloDriverStartHandler))
.SetJobIdentifier("HelloDriver")
.Build();
IJobSubmissionResult jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(helloJobSubmission);
return(jobSubmissionResult);
}
/// <summary>
/// Runs HelloREEF using the IREEFClient passed into the constructor.
/// </summary>
private void Run()
{
// The driver configuration contains all the needed handler bindings
var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloDriverYarn> .Class)
.Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriverYarn> .Class)
.Build();
var driverConfig = TangFactory.GetTang()
.NewConfigurationBuilder(helloDriverConfiguration);
foreach (var n in _nodeNames)
{
driverConfig.BindSetEntry <NodeNames, string>(GenericType <NodeNames> .Class, n);
}
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
var helloJobRequest = _jobRequestBuilder
.AddDriverConfiguration(driverConfig.Build())
.AddGlobalAssemblyForType(typeof(HelloDriverYarn))
.SetJobIdentifier("HelloREEF")
.SetJavaLogLevel(JavaLoggingSetting.Verbose)
.Build();
var result = _reefClient.SubmitAndGetJobStatus(helloJobRequest);
LogApplicationReport();
//// This is an example to Kill Job Application
//// KillApplication(result.AppId);
var state = PullFinalJobStatus(result);
Logger.Log(Level.Info, "Application final state : {0}.", state);
}
/// <summary>
/// Runs DriverRestart using the IREEFClient passed into the constructor.
/// </summary>
private void Run()
{
// The driver configuration contains all the needed bindings.
var driverConfiguration = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnDriverStarted, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnDriverRestarted, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnContextActive, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnTaskRunning, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnTaskFailed, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnTaskCompleted, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnDriverRestartCompleted, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnDriverRestartContextActive, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnDriverRestartTaskRunning, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnEvaluatorFailed, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnDriverRestartEvaluatorFailed, GenericType <HelloRestartDriver> .Class)
.Set(DriverConfiguration.OnDriverReconnect, GenericType <DefaultYarnClusterHttpDriverConnection> .Class)
.Set(DriverConfiguration.DriverRestartEvaluatorRecoverySeconds, (5 * 60).ToString())
.Build();
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
var restartJobSubmission = _jobRequestBuilder
.AddDriverConfiguration(driverConfiguration)
.AddGlobalAssemblyForType(typeof(HelloRestartDriver))
.SetJobIdentifier("DriverRestart_" + Guid.NewGuid().ToString().Substring(0, 6))
.SetMaxApplicationSubmissions(2)
.Build();
_reefClient.SubmitAndGetJobStatus(restartJobSubmission);
}
public ITestResult RunTest(JobRequestBuilder jobRequestBuilder)
{
// Setup the assert file.
var assertFileName = Path.GetTempPath() + "/reef-test-" + DateTime.Now.ToString("yyyyMMddHHmmssfff") + ".json";
jobRequestBuilder.AddDriverConfiguration(FileWritingAssertConfiguration.ConfigurationModule
.Set(FileWritingAssertConfiguration.FilePath, assertFileName)
.Build());
var jobRequest = jobRequestBuilder.Build();
LOG.Log(Level.Info, "Submitting job `{0}` for execution. Assert log in `{1}`",
jobRequest.JobIdentifier,
assertFileName);
IJobSubmissionResult jobStatus = _client.SubmitAndGetJobStatus(jobRequest);
if (jobStatus == null)
{
return(TestResult.Fail(
"JobStatus returned by the Client was null. This points to an environment setup problem."));
}
LOG.Log(Level.Verbose, "Waiting for job `{0}` to complete.", jobRequest.JobIdentifier);
jobStatus.WaitForDriverToFinish();
LOG.Log(Level.Verbose, "Job `{0}` completed.", jobRequest.JobIdentifier);
return(ReadTestResult(assertFileName));
}
/// <summary>
/// Runs HelloREEF using the IREEFClient passed into the constructor.
/// </summary>
private void Run()
{
// The driver configuration contains all the needed bindings.
var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloDriver> .Class)
.Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriver> .Class)
.Build();
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
var helloJobRequest = _jobRequestBuilder
.AddDriverConfiguration(helloDriverConfiguration)
.AddGlobalAssemblyForType(typeof(HelloDriver))
.SetJobIdentifier("HelloREEF")
.Build();
_reefClient.Submit(helloJobRequest);
}
/// <summary>
/// Runs HelloREEF using the IREEFClient passed into the constructor.
/// </summary>
private void Run()
{
// The driver configuration contains all the needed bindings.
var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <Driver> .Class)
.Set(DriverConfiguration.OnDriverStarted, GenericType <Driver> .Class)
.Set(DriverConfiguration.OnEvaluatorFailed, GenericType <Driver> .Class)
.Set(DriverConfiguration.OnTaskFailed, GenericType <Driver> .Class)
.Build();
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
var helloJobRequest = _jobRequestBuilder
.AddDriverConfiguration(helloDriverConfiguration)
.AddGlobalAssemblyForType(typeof(Driver))
.AddGlobalFile(Common.GlobalConfigFileName)
.SetJobIdentifier(Utils.ReadStringArgumentFromConf(Common.GlobalConfigTaskName))
.Build();
_reefClient.Submit(helloJobRequest);
}
private IJobSubmissionResult Run()
{
var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloAllocatedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <AnotherHelloAllocatedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnContextActive, GenericType <HelloActiveContextHandler> .Class)
.Set(DriverConfiguration.OnTaskMessage, GenericType <HelloTaskMessageHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorFailed, GenericType <HelloFailedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnTaskFailed, GenericType <HelloFailedTaskHandler> .Class)
.Set(DriverConfiguration.OnTaskRunning, GenericType <HelloRunningTaskHandler> .Class)
.Set(DriverConfiguration.OnTaskCompleted, GenericType <HelloTaskCompletedHandler> .Class)
.Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriverStartHandler> .Class)
.Set(DriverConfiguration.OnHttpEvent, GenericType <HelloHttpHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorCompleted, GenericType <HelloCompletedEvaluatorHandler> .Class)
.Set(DriverConfiguration.CustomTraceListeners, GenericType <DefaultCustomTraceListener> .Class)
.Set(DriverConfiguration.CustomTraceLevel, Level.Info.ToString())
.Set(DriverConfiguration.OnDriverRestarted, GenericType <HelloRestartHandler> .Class)
.Set(DriverConfiguration.OnDriverReconnect, GenericType <DefaultLocalHttpDriverConnection> .Class)
.Set(DriverConfiguration.OnDriverRestartContextActive, GenericType <HelloDriverRestartActiveContextHandler> .Class)
.Set(DriverConfiguration.OnDriverRestartTaskRunning, GenericType <HelloDriverRestartRunningTaskHandler> .Class)
.Build();
var driverConfig = TangFactory.GetTang().NewConfigurationBuilder(helloDriverConfiguration)
.BindSetEntry <DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(HelloTask).Assembly.GetName().Name)
.BindSetEntry <DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(NameClient).Assembly.GetName().Name)
.Build();
var helloJobSubmission = _jobRequestBuilder
.AddDriverConfiguration(driverConfig)
.AddGlobalAssemblyForType(typeof(HelloDriverStartHandler))
.SetJobIdentifier("HelloDriver")
.Build();
IJobSubmissionResult jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(helloJobSubmission);
return(jobSubmissionResult);
}
/// <summary>
/// Submits the job to reefClient
/// </summary>
/// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam>
/// <typeparam name="TMapOutput">The return type of the Map function</typeparam>
/// <typeparam name="TResult">The return type of the computation.</typeparam>
/// <typeparam name="TPartitionType">Type of data partition (Generic type in IInputPartition)</typeparam>
/// <param name="jobDefinition">IMRU job definition given by the user</param>
/// <returns>Null as results will be later written to some directory</returns>
IEnumerable <TResult> IIMRUClient.Submit <TMapInput, TMapOutput, TResult, TPartitionType>(IMRUJobDefinition jobDefinition)
{
string driverId = string.Format("IMRU-{0}-Driver", jobDefinition.JobName);
IConfiguration overallPerMapConfig = null;
try
{
overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray());
}
catch (Exception e)
{
Exceptions.Throw(e, "Issues in merging PerMapCOnfigGenerator configurations", Logger);
}
// The driver configuration contains all the needed bindings.
var imruDriverConfiguration = TangFactory.GetTang().NewConfigurationBuilder(new[]
{
DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnDriverStarted,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnContextActive,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnTaskCompleted,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnEvaluatorFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnContextFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnTaskFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.CustomTraceLevel, TraceLevel.Info.ToString())
.Build(),
TangFactory.GetTang().NewConfigurationBuilder()
.BindStringNamedParam <GroupCommConfigurationOptions.DriverId>(driverId)
.BindStringNamedParam <GroupCommConfigurationOptions.MasterTaskId>(IMRUConstants.UpdateTaskName)
.BindStringNamedParam <GroupCommConfigurationOptions.GroupName>(IMRUConstants.CommunicationGroupName)
.BindIntNamedParam <GroupCommConfigurationOptions.FanOut>(
IMRUConstants.TreeFanout.ToString(CultureInfo.InvariantCulture)
.ToString(CultureInfo.InvariantCulture))
.BindIntNamedParam <GroupCommConfigurationOptions.NumberOfTasks>(
(jobDefinition.NumberOfMappers + 1).ToString(CultureInfo.InvariantCulture))
.BindImplementation(GenericType <IGroupCommDriver> .Class, GenericType <GroupCommDriver> .Class)
.Build(),
jobDefinition.PartitionedDatasetConfiguration,
overallPerMapConfig
})
.BindNamedParameter(typeof(SerializedMapConfiguration),
_configurationSerializer.ToString(jobDefinition.MapFunctionConfiguration))
.BindNamedParameter(typeof(SerializedUpdateConfiguration),
_configurationSerializer.ToString(jobDefinition.UpdateFunctionConfiguration))
.BindNamedParameter(typeof(SerializedMapInputCodecConfiguration),
_configurationSerializer.ToString(jobDefinition.MapInputCodecConfiguration))
.BindNamedParameter(typeof(SerializedMapInputPipelineDataConverterConfiguration),
_configurationSerializer.ToString(jobDefinition.MapInputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedUpdateFunctionCodecsConfiguration),
_configurationSerializer.ToString(jobDefinition.UpdateFunctionCodecsConfiguration))
.BindNamedParameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration),
_configurationSerializer.ToString(jobDefinition.MapOutputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedReduceConfiguration),
_configurationSerializer.ToString(jobDefinition.ReduceFunctionConfiguration))
.BindNamedParameter(typeof(SerializedResultHandlerConfiguration),
_configurationSerializer.ToString(jobDefinition.ResultHandlerConfiguration))
.BindNamedParameter(typeof(MemoryPerMapper),
jobDefinition.MapperMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(MemoryForUpdateTask),
jobDefinition.UpdateTaskMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresPerMapper),
jobDefinition.MapTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresForUpdateTask),
jobDefinition.UpdateTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(InvokeGC),
jobDefinition.InvokeGarbageCollectorAfterIteration.ToString(CultureInfo.InvariantCulture))
.Build();
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
var imruJobSubmission = _jobRequestBuilder
.AddDriverConfiguration(imruDriverConfiguration)
.AddGlobalAssemblyForType(typeof(IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType>))
.SetJobIdentifier(jobDefinition.JobName)
.SetDriverMemory(5000)
.Build();
_jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(imruJobSubmission);
return(null);
}