/// <summary> /// Runs DriverRestart using the IREEFClient passed into the constructor. /// </summary> private void Run() { // The driver configuration contains all the needed bindings. var driverConfiguration = DriverConfiguration.ConfigurationModule .Set(DriverConfiguration.OnDriverStarted, GenericType <HelloRestartDriver> .Class) .Set(DriverConfiguration.OnDriverRestarted, GenericType <HelloRestartDriver> .Class) .Set(DriverConfiguration.OnContextActive, GenericType <HelloRestartDriver> .Class) .Set(DriverConfiguration.OnTaskRunning, GenericType <HelloRestartDriver> .Class) .Set(DriverConfiguration.OnTaskFailed, GenericType <HelloRestartDriver> .Class) .Set(DriverConfiguration.OnTaskCompleted, GenericType <HelloRestartDriver> .Class) .Set(DriverConfiguration.OnDriverRestartCompleted, GenericType <HelloRestartDriver> .Class) .Set(DriverConfiguration.OnDriverRestartContextActive, GenericType <HelloRestartDriver> .Class) .Set(DriverConfiguration.OnDriverRestartTaskRunning, GenericType <HelloRestartDriver> .Class) .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloRestartDriver> .Class) .Set(DriverConfiguration.OnEvaluatorFailed, GenericType <HelloRestartDriver> .Class) .Set(DriverConfiguration.OnDriverRestartEvaluatorFailed, GenericType <HelloRestartDriver> .Class) .Set(DriverConfiguration.OnDriverReconnect, GenericType <DefaultYarnClusterHttpDriverConnection> .Class) .Set(DriverConfiguration.DriverRestartEvaluatorRecoverySeconds, (5 * 60).ToString()) .Set(DriverConfiguration.MaxApplicationSubmissions, 2.ToString()) .Build(); // The JobSubmission contains the Driver configuration as well as the files needed on the Driver. var restartJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder() .AddDriverConfiguration(driverConfiguration) .AddGlobalAssemblyForType(typeof(HelloRestartDriver)) .SetJobIdentifier("DriverRestart_" + Guid.NewGuid().ToString().Substring(0, 6)) .Build(); _reefClient.SubmitAndGetJobStatus(restartJobSubmission); }
/// <summary> /// Runs HelloREEF using the IREEFClient passed into the constructor. /// </summary> private void Run() { // The driver configuration contains all the needed bindings. var helloDriverConfiguration = DriverConfiguration.ConfigurationModule .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloDriver> .Class) .Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriver> .Class) .Build(); // The JobSubmission contains the Driver configuration as well as the files needed on the Driver. var helloJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder() .AddDriverConfiguration(helloDriverConfiguration) .AddGlobalAssemblyForType(typeof(HelloDriver)) .SetJobIdentifier("HelloREEF") .Build(); _reefClient.Submit(helloJobSubmission); }
private IJobSubmissionResult Run() { var helloDriverConfiguration = DriverConfiguration.ConfigurationModule .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloAllocatedEvaluatorHandler> .Class) .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <AnotherHelloAllocatedEvaluatorHandler> .Class) .Set(DriverConfiguration.OnContextActive, GenericType <HelloActiveContextHandler> .Class) .Set(DriverConfiguration.OnTaskMessage, GenericType <HelloTaskMessageHandler> .Class) .Set(DriverConfiguration.OnEvaluatorFailed, GenericType <HelloFailedEvaluatorHandler> .Class) .Set(DriverConfiguration.OnTaskFailed, GenericType <HelloFailedTaskHandler> .Class) .Set(DriverConfiguration.OnTaskRunning, GenericType <HelloRunningTaskHandler> .Class) .Set(DriverConfiguration.OnTaskCompleted, GenericType <HelloTaskCompletedHandler> .Class) .Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriverStartHandler> .Class) .Set(DriverConfiguration.OnHttpEvent, GenericType <HelloHttpHandler> .Class) .Set(DriverConfiguration.OnEvaluatorCompleted, GenericType <HelloCompletedEvaluatorHandler> .Class) .Set(DriverConfiguration.CustomTraceListeners, GenericType <DefaultCustomTraceListener> .Class) .Set(DriverConfiguration.CustomTraceLevel, Level.Info.ToString()) .Set(DriverConfiguration.OnDriverRestarted, GenericType <HelloRestartHandler> .Class) .Set(DriverConfiguration.OnDriverReconnect, GenericType <DefaultLocalHttpDriverConnection> .Class) .Set(DriverConfiguration.OnDriverRestartContextActive, GenericType <HelloDriverRestartActiveContextHandler> .Class) .Set(DriverConfiguration.OnDriverRestartTaskRunning, GenericType <HelloDriverRestartRunningTaskHandler> .Class) .Build(); var driverConfig = TangFactory.GetTang().NewConfigurationBuilder(helloDriverConfiguration) .BindSetEntry <DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(HelloTask).Assembly.GetName().Name) .BindSetEntry <DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(NameClient).Assembly.GetName().Name) .Build(); var helloJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder() .AddDriverConfiguration(driverConfig) .AddGlobalAssemblyForType(typeof(HelloDriverStartHandler)) .SetJobIdentifier("HelloDriver") .Build(); IJobSubmissionResult jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(helloJobSubmission); return(jobSubmissionResult); }
/// <summary> /// Submits the job to reefClient /// </summary> /// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam> /// <typeparam name="TMapOutput">The return type of the Map function</typeparam> /// <typeparam name="TResult">The return type of the computation.</typeparam> /// <param name="jobDefinition">IMRU job definition given by the user</param> /// <returns>Null as results will be later written to some directory</returns> IEnumerable <TResult> IIMRUClient.Submit <TMapInput, TMapOutput, TResult>(IMRUJobDefinition jobDefinition) { string driverId = string.Format("IMRU-{0}-Driver", jobDefinition.JobName); IConfiguration overallPerMapConfig = null; try { overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray()); } catch (Exception e) { Exceptions.Throw(e, "Issues in merging PerMapCOnfigGenerator configurations", Logger); } // The driver configuration contains all the needed bindings. var imruDriverConfiguration = TangFactory.GetTang().NewConfigurationBuilder(new[] { DriverConfiguration.ConfigurationModule .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult> > .Class) .Set(DriverConfiguration.OnDriverStarted, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult> > .Class) .Set(DriverConfiguration.OnContextActive, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult> > .Class) .Set(DriverConfiguration.OnTaskCompleted, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult> > .Class) .Set(DriverConfiguration.OnEvaluatorFailed, GenericType <IMRUDriver <TMapInput, TMapOutput, TResult> > .Class) .Set(DriverConfiguration.CustomTraceLevel, TraceLevel.Info.ToString()) .Build(), TangFactory.GetTang().NewConfigurationBuilder() .BindStringNamedParam <GroupCommConfigurationOptions.DriverId>(driverId) .BindStringNamedParam <GroupCommConfigurationOptions.MasterTaskId>(IMRUConstants.UpdateTaskName) .BindStringNamedParam <GroupCommConfigurationOptions.GroupName>(IMRUConstants.CommunicationGroupName) .BindIntNamedParam <GroupCommConfigurationOptions.FanOut>( IMRUConstants.TreeFanout.ToString(CultureInfo.InvariantCulture) .ToString(CultureInfo.InvariantCulture)) .BindIntNamedParam <GroupCommConfigurationOptions.NumberOfTasks>( (jobDefinition.NumberOfMappers + 1).ToString(CultureInfo.InvariantCulture)) .BindImplementation(GenericType <IGroupCommDriver> .Class, GenericType <GroupCommDriver> .Class) .Build(), jobDefinition.PartitionedDatasetConfiguration, overallPerMapConfig }) .BindNamedParameter(typeof(SerializedMapConfiguration), _configurationSerializer.ToString(jobDefinition.MapFunctionConfiguration)) .BindNamedParameter(typeof(SerializedUpdateConfiguration), _configurationSerializer.ToString(jobDefinition.UpdateFunctionConfiguration)) .BindNamedParameter(typeof(SerializedMapInputCodecConfiguration), _configurationSerializer.ToString(jobDefinition.MapInputCodecConfiguration)) .BindNamedParameter(typeof(SerializedMapInputPipelineDataConverterConfiguration), _configurationSerializer.ToString(jobDefinition.MapInputPipelineDataConverterConfiguration)) .BindNamedParameter(typeof(SerializedUpdateFunctionCodecsConfiguration), _configurationSerializer.ToString(jobDefinition.UpdateFunctionCodecsConfiguration)) .BindNamedParameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration), _configurationSerializer.ToString(jobDefinition.MapOutputPipelineDataConverterConfiguration)) .BindNamedParameter(typeof(SerializedReduceConfiguration), _configurationSerializer.ToString(jobDefinition.ReduceFunctionConfiguration)) .BindNamedParameter(typeof(SerializedResultHandlerConfiguration), _configurationSerializer.ToString(jobDefinition.ResultHandlerConfiguration)) .BindNamedParameter(typeof(MemoryPerMapper), jobDefinition.MapperMemory.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(MemoryForUpdateTask), jobDefinition.UpdateTaskMemory.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(CoresPerMapper), jobDefinition.MapTaskCores.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(CoresForUpdateTask), jobDefinition.UpdateTaskCores.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter(typeof(InvokeGC), jobDefinition.InvokeGarbageCollectorAfterIteration.ToString(CultureInfo.InvariantCulture)) .Build(); // The JobSubmission contains the Driver configuration as well as the files needed on the Driver. var imruJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder() .AddDriverConfiguration(imruDriverConfiguration) .AddGlobalAssemblyForType(typeof(IMRUDriver <TMapInput, TMapOutput, TResult>)) .SetJobIdentifier(jobDefinition.JobName) .Build(); _jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(imruJobSubmission); return(null); }