private IJobSubmissionResult Run()
{
var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloAllocatedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <AnotherHelloAllocatedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnContextActive, GenericType <HelloActiveContextHandler> .Class)
.Set(DriverConfiguration.OnTaskMessage, GenericType <HelloTaskMessageHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorFailed, GenericType <HelloFailedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnTaskFailed, GenericType <HelloFailedTaskHandler> .Class)
.Set(DriverConfiguration.OnTaskRunning, GenericType <HelloRunningTaskHandler> .Class)
.Set(DriverConfiguration.OnTaskCompleted, GenericType <HelloTaskCompletedHandler> .Class)
.Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriverStartHandler> .Class)
.Set(DriverConfiguration.OnHttpEvent, GenericType <HelloHttpHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorCompleted, GenericType <HelloCompletedEvaluatorHandler> .Class)
.Set(DriverConfiguration.CustomTraceListeners, GenericType <DefaultCustomTraceListener> .Class)
.Set(DriverConfiguration.CustomTraceLevel, Level.Info.ToString())
.Set(DriverConfiguration.OnDriverRestarted, GenericType <HelloRestartHandler> .Class)
.Set(DriverConfiguration.DriverReconnectionConfigurationProvider,
GenericType <LocalHttpDriverReconnConfigProvider> .Class)
.Set(DriverConfiguration.OnDriverRestartContextActive, GenericType <HelloDriverRestartActiveContextHandler> .Class)
.Set(DriverConfiguration.OnDriverRestartTaskRunning, GenericType <HelloDriverRestartRunningTaskHandler> .Class)
.Build();
var helloJobSubmission = _jobRequestBuilder
.AddDriverConfiguration(helloDriverConfiguration)
.AddGlobalAssemblyForType(typeof(HelloDriverStartHandler))
.SetJobIdentifier("HelloDriver")
.Build();
IJobSubmissionResult jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(helloJobSubmission);
return(jobSubmissionResult);
}
private async Task RunClrBridgeClient(bool runOnYarn, string testRuntimeFolder)
{
string[] a = { runOnYarn ? "yarn" : "local", testRuntimeFolder };
IJobSubmissionResult driverHttpEndpoint = AllHandlers.Run(a);
var driverUrl = driverHttpEndpoint.DriverUrl;
int retryCount = 1;
while (string.IsNullOrEmpty(driverUrl) && retryCount < 10)
{
driverUrl = driverHttpEndpoint.DriverUrl;
retryCount++;
}
if (driverUrl != null)
{
var uri = driverUrl + "NRT/status?a=1&b=2";
var strStatus = driverHttpEndpoint.GetUrlResult(uri);
Assert.NotNull(strStatus);
Assert.True(strStatus.Equals("Byte array returned from HelloHttpHandler in CLR!!!\r\n"));
var uri1 = driverUrl + "CMD/Stop?c=1";
var strStatus1 = driverHttpEndpoint.GetUrlResult(uri1);
Assert.NotNull(strStatus1);
Assert.True(strStatus1.Equals("Stopped!!!\r\n"));
await((JobSubmissionResult)driverHttpEndpoint).TryUntilNoConnection(uri);
}
}
/// <summary>
/// Runs HelloREEF using the IREEFClient passed into the constructor.
/// </summary>
private void Run()
{
// The driver configuration contains all the needed bindings.
var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloDriver> .Class)
.Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriver> .Class)
.Set(DriverConfiguration.CustomTraceLevel, Level.Verbose.ToString())
.Build();
string applicationId = GetApplicationId();
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
var helloJobRequest = _reefClient.NewJobRequestBuilder()
.AddDriverConfiguration(helloDriverConfiguration)
.AddGlobalAssemblyForType(typeof(HelloDriver))
.AddGlobalAssembliesInDirectoryOfExecutingAssembly()
.SetJobIdentifier(applicationId)
.SetJavaLogLevel(JavaLoggingSetting.Verbose)
.Build();
IJobSubmissionResult jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(helloJobRequest);
// Wait for the Driver to complete.
jobSubmissionResult?.WaitForDriverToFinish();
}
public ITestResult RunTest(JobRequestBuilder jobRequestBuilder)
{
// Setup the assert file.
var assertFileName = Path.GetTempPath() + "/reef-test-" + DateTime.Now.ToString("yyyyMMddHHmmssfff") + ".json";
jobRequestBuilder.AddDriverConfiguration(FileWritingAssertConfiguration.ConfigurationModule
.Set(FileWritingAssertConfiguration.FilePath, assertFileName)
.Build());
var jobRequest = jobRequestBuilder.Build();
LOG.Log(Level.Info, "Submitting job `{0}` for execution. Assert log in `{1}`",
jobRequest.JobIdentifier,
assertFileName);
IJobSubmissionResult jobStatus = _client.SubmitAndGetJobStatus(jobRequest);
if (jobStatus == null)
{
return(TestResult.Fail(
"JobStatus returned by the Client was null. This points to an environment setup problem."));
}
LOG.Log(Level.Verbose, "Waiting for job `{0}` to complete.", jobRequest.JobIdentifier);
jobStatus.WaitForDriverToFinish();
LOG.Log(Level.Verbose, "Job `{0}` completed.", jobRequest.JobIdentifier);
return(ReadTestResult(assertFileName));
}
/// <summary>
/// </summary>
/// <param name="args"></param>
/// <remarks>
/// args[0] specify either running local or YARN. Default is local
/// args[1] specify running folder. Default is REEF_LOCAL_RUNTIME
/// </remarks>
public static IJobSubmissionResult Run(string[] args)
{
string runOnYarn = args.Length > 0 ? args[0] : Local;
string runtimeFolder = args.Length > 1 ? args[1] : "REEF_LOCAL_RUNTIME";
IJobSubmissionResult jobSubmissionResult = TangFactory.GetTang().NewInjector(GetRuntimeConfiguration(runOnYarn, runtimeFolder)).GetInstance <AllHandlers>().Run();
return(jobSubmissionResult);
}
/// <summary>
/// Sample code to pull job final status until the Job is done
/// </summary>
/// <param name="jobSubmitionResult"></param>
/// <returns></returns>
private FinalState PullFinalJobStatus(IJobSubmissionResult jobSubmitionResult)
{
int n = 0;
var state = jobSubmitionResult.FinalState;
while (state.Equals(FinalState.UNDEFINED) && n++ < ReTryCounts)
{
Thread.Sleep(SleepTime);
state = jobSubmitionResult.FinalState;
}
return(state);
}
private async Task RunClrBridgeClient(bool runOnYarn, string testRuntimeFolder)
{
string[] a = { runOnYarn ? "yarn" : "local", testRuntimeFolder };
IJobSubmissionResult driverHttpEndpoint = AllHandlers.Run(a);
var uri = driverHttpEndpoint.DriverUrl + "NRT/status?a=1&b=2";
var strStatus = driverHttpEndpoint.GetUrlResult(uri);
Assert.NotNull(strStatus);
Assert.True(strStatus.Equals("Byte array returned from HelloHttpHandler in CLR!!!\r\n"));
await((JobSubmissionResult)driverHttpEndpoint).TryUntilNoConnection(uri);
}
private async void RunClrBridgeClient(bool runOnYarn, string testRuntimeFolder)
{
string[] a = new[] { runOnYarn ? "yarn" : "local", testRuntimeFolder };
IJobSubmissionResult driverHttpEndpoint = AllHandlers.Run(a);
var uri = driverHttpEndpoint.DriverUrl + "NRT/status?a=1&b=2";
var strStatus = driverHttpEndpoint.GetUrlResult(uri);
Assert.IsTrue(strStatus.Equals("Byte array returned from HelloHttpHandler in CLR!!!\r\n"));
await((JobSubmissionResult)driverHttpEndpoint).TryUntilNoConnection(uri);
ValidateSuccessForLocalRuntime(2, testFolder: testRuntimeFolder);
CleanUp(testRuntimeFolder);
}
private IJobSubmissionResult Run()
{
var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloAllocatedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <AnotherHelloAllocatedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnContextActive, GenericType <HelloActiveContextHandler> .Class)
.Set(DriverConfiguration.OnTaskMessage, GenericType <HelloTaskMessageHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorFailed, GenericType <HelloFailedEvaluatorHandler> .Class)
.Set(DriverConfiguration.OnTaskFailed, GenericType <HelloFailedTaskHandler> .Class)
.Set(DriverConfiguration.OnTaskRunning, GenericType <HelloRunningTaskHandler> .Class)
.Set(DriverConfiguration.OnTaskCompleted, GenericType <HelloTaskCompletedHandler> .Class)
.Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriverStartHandler> .Class)
.Set(DriverConfiguration.OnHttpEvent, GenericType <HelloHttpHandler> .Class)
.Set(DriverConfiguration.OnEvaluatorCompleted, GenericType <HelloCompletedEvaluatorHandler> .Class)
.Set(DriverConfiguration.CustomTraceListeners, GenericType <DefaultCustomTraceListener> .Class)
.Set(DriverConfiguration.CustomTraceLevel, Level.Info.ToString())
.Set(DriverConfiguration.OnDriverRestarted, GenericType <HelloRestartHandler> .Class)
.Set(DriverConfiguration.OnDriverReconnect, GenericType <DefaultLocalHttpDriverConnection> .Class)
.Set(DriverConfiguration.OnDriverRestartContextActive, GenericType <HelloDriverRestartActiveContextHandler> .Class)
.Set(DriverConfiguration.OnDriverRestartTaskRunning, GenericType <HelloDriverRestartRunningTaskHandler> .Class)
.Build();
var driverConfig = TangFactory.GetTang().NewConfigurationBuilder(helloDriverConfiguration)
.BindSetEntry <DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(HelloTask).Assembly.GetName().Name)
.BindSetEntry <DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(NameClient).Assembly.GetName().Name)
.Build();
var helloJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder()
.AddDriverConfiguration(driverConfig)
.AddGlobalAssemblyForType(typeof(HelloDriverStartHandler))
.SetJobIdentifier("HelloDriver")
.Build();
IJobSubmissionResult jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(helloJobSubmission);
return(jobSubmissionResult);
}
/// <summary>
/// Submits the job to reefClient
/// </summary>
/// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam>
/// <typeparam name="TMapOutput">The return type of the Map function</typeparam>
/// <typeparam name="TResult">The return type of the computation.</typeparam>
/// <typeparam name="TPartitionType">Type of data partition (Generic type in IInputPartition)</typeparam>
/// <param name="jobDefinition">IMRU job definition given by the user</param>
/// <returns>Null as results will be later written to some directory</returns>
IEnumerable <TResult> IIMRUClient.Submit <TMapInput, TMapOutput, TResult, TPartitionType>(IMRUJobDefinition jobDefinition)
{
string driverId = string.Format("IMRU-{0}-Driver", jobDefinition.JobName);
IConfiguration overallPerMapConfig = null;
try
{
overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray());
}
catch (Exception e)
{
Exceptions.Throw(e, "Issues in merging PerMapCOnfigGenerator configurations", Logger);
}
// The driver configuration contains all the needed bindings.
var imruDriverConfiguration = TangFactory.GetTang().NewConfigurationBuilder(new[]
{
DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnEvaluatorAllocated,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnDriverStarted,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnContextActive,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnTaskCompleted,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnEvaluatorFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnContextFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.OnTaskFailed,
GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
.Set(DriverConfiguration.CustomTraceLevel, TraceLevel.Info.ToString())
.Build(),
TangFactory.GetTang().NewConfigurationBuilder()
.BindStringNamedParam <GroupCommConfigurationOptions.DriverId>(driverId)
.BindStringNamedParam <GroupCommConfigurationOptions.MasterTaskId>(IMRUConstants.UpdateTaskName)
.BindStringNamedParam <GroupCommConfigurationOptions.GroupName>(IMRUConstants.CommunicationGroupName)
.BindIntNamedParam <GroupCommConfigurationOptions.FanOut>(
IMRUConstants.TreeFanout.ToString(CultureInfo.InvariantCulture)
.ToString(CultureInfo.InvariantCulture))
.BindIntNamedParam <GroupCommConfigurationOptions.NumberOfTasks>(
(jobDefinition.NumberOfMappers + 1).ToString(CultureInfo.InvariantCulture))
.BindImplementation(GenericType <IGroupCommDriver> .Class, GenericType <GroupCommDriver> .Class)
.Build(),
jobDefinition.PartitionedDatasetConfiguration,
overallPerMapConfig
})
.BindNamedParameter(typeof(SerializedMapConfiguration),
_configurationSerializer.ToString(jobDefinition.MapFunctionConfiguration))
.BindNamedParameter(typeof(SerializedUpdateConfiguration),
_configurationSerializer.ToString(jobDefinition.UpdateFunctionConfiguration))
.BindNamedParameter(typeof(SerializedMapInputCodecConfiguration),
_configurationSerializer.ToString(jobDefinition.MapInputCodecConfiguration))
.BindNamedParameter(typeof(SerializedMapInputPipelineDataConverterConfiguration),
_configurationSerializer.ToString(jobDefinition.MapInputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedUpdateFunctionCodecsConfiguration),
_configurationSerializer.ToString(jobDefinition.UpdateFunctionCodecsConfiguration))
.BindNamedParameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration),
_configurationSerializer.ToString(jobDefinition.MapOutputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedReduceConfiguration),
_configurationSerializer.ToString(jobDefinition.ReduceFunctionConfiguration))
.BindNamedParameter(typeof(SerializedResultHandlerConfiguration),
_configurationSerializer.ToString(jobDefinition.ResultHandlerConfiguration))
.BindNamedParameter(typeof(MemoryPerMapper),
jobDefinition.MapperMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(MemoryForUpdateTask),
jobDefinition.UpdateTaskMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresPerMapper),
jobDefinition.MapTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresForUpdateTask),
jobDefinition.UpdateTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(InvokeGC),
jobDefinition.InvokeGarbageCollectorAfterIteration.ToString(CultureInfo.InvariantCulture))
.Build();
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
var imruJobSubmission = _jobRequestBuilder
.AddDriverConfiguration(imruDriverConfiguration)
.AddGlobalAssemblyForType(typeof(IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType>))
.SetJobIdentifier(jobDefinition.JobName)
.SetDriverMemory(5000)
.Build();
_jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(imruJobSubmission);
return(null);
}
