public void Submit(JobRequest jobRequest)
{
// Prepare the job submission folder
var driverFolderPath = CreateDriverFolder(jobRequest.JobIdentifier);
Logger.Log(Level.Verbose, "Preparing driver folder in " + driverFolderPath);
Launch(jobRequest, driverFolderPath);
}
internal JobSubmission(JobRequest jobRequest)
{
_driverConfigurations = jobRequest.DriverConfigurations;
_globalAssemblies = jobRequest.GlobalAssemblies;
_globalFiles = jobRequest.GlobalFiles;
_localAssemblies = jobRequest.LocalAssemblies;
_localFiles = jobRequest.LocalFiles;
_driverMemory = jobRequest.DriverMemory;
_jobIdentifier = jobRequest.JobIdentifier;
_driverConfigurationFileContents = jobRequest.DriverConfigurationFileContents;
_maxAppSubmissions = jobRequest.MaxApplicationSubmissions;
}
internal JobSubmission(JobRequest jobRequest)
{
_driverConfigurations = jobRequest.DriverConfigurations;
_globalAssemblies = jobRequest.GlobalAssemblies;
_globalFiles = jobRequest.GlobalFiles;
_localAssemblies = jobRequest.LocalAssemblies;
_localFiles = jobRequest.LocalFiles;
_driverMemory = jobRequest.DriverMemory;
_jobIdentifier = jobRequest.JobIdentifier;
_driverConfigurationFileContents = jobRequest.DriverConfigurationFileContents;
_maxAppSubmissions = jobRequest.MaxApplicationSubmissions;
}
public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest)
{
// Prepare the job submission folder
var driverFolderPath = CreateDriverFolder(jobRequest.JobIdentifier);
Logger.Log(Level.Verbose, "Preparing driver folder in " + driverFolderPath);
Launch(jobRequest, driverFolderPath);
var pointerFileName = Path.Combine(driverFolderPath, _fileNames.DriverHttpEndpoint);
var jobSubmitionResultImpl = new YarnJobSubmissionResult(this, pointerFileName);
var msg = string.Format(CultureInfo.CurrentCulture,
"Submitted the Driver for execution. Returned driverUrl is: {0}, appId is {1}.",
jobSubmitionResultImpl.DriverUrl, jobSubmitionResultImpl.AppId);
Logger.Log(Level.Info, msg);
return jobSubmitionResultImpl;
}
private void Launch(JobRequest jobRequest, string driverFolderPath)
{
_driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, driverFolderPath);
// TODO: Remove this when we have a generalized way to pass config to java
var paramInjector = TangFactory.GetTang().NewInjector(jobRequest.DriverConfigurations.ToArray());
var submissionJobArgsFilePath = _paramSerializer.SerializeJobFile(jobRequest.JobParameters, paramInjector, driverFolderPath);
var submissionAppArgsFilePath = _paramSerializer.SerializeAppFile(jobRequest.AppParameters, paramInjector, driverFolderPath);
// Submit the driver
_javaClientLauncher.LaunchAsync(
jobRequest.JavaLogLevel, JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath)
.GetAwaiter()
.GetResult();
Logger.Log(Level.Info, "Submitted the Driver for execution." + jobRequest.JobIdentifier);
}
public void Submit(JobRequest jobRequest)
{
string jobId = jobRequest.JobIdentifier;
// todo: Future client interface should be async.
// Using GetAwaiter().GetResult() instead of .Result to avoid exception
// getting wrapped in AggregateException.
var newApplication = _yarnRMClient.CreateNewApplicationAsync().GetAwaiter().GetResult();
string applicationId = newApplication.ApplicationId;
// create job submission remote path
string jobSubmissionDirectory =
_jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory(applicationId);
// create local driver folder.
var localDriverFolderPath = CreateDriverFolder(jobId, applicationId);
try
{
Log.Log(Level.Verbose, "Preparing driver folder in {0}", localDriverFolderPath);
_driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, localDriverFolderPath);
// prepare configuration
var paramInjector = TangFactory.GetTang().NewInjector(jobRequest.DriverConfigurations.ToArray());
_paramSerializer.SerializeAppFile(jobRequest.AppParameters, paramInjector, localDriverFolderPath);
_paramSerializer.SerializeJobFile(jobRequest.JobParameters, localDriverFolderPath, jobSubmissionDirectory);
var archiveResource =
_jobResourceUploader.UploadArchiveResourceAsync(localDriverFolderPath, jobSubmissionDirectory)
.GetAwaiter()
.GetResult();
// Path to the job args file.
var jobArgsFilePath = Path.Combine(localDriverFolderPath, _fileNames.GetJobSubmissionParametersFile());
var argFileResource =
_jobResourceUploader.UploadFileResourceAsync(jobArgsFilePath, jobSubmissionDirectory)
.GetAwaiter()
.GetResult();
// upload prepared folder to DFS
var jobResources = new List<JobResource> { archiveResource, argFileResource };
// submit job
Log.Log(Level.Verbose, @"Assigned application id {0}", applicationId);
var submissionReq = CreateApplicationSubmissionRequest(
jobRequest.JobParameters,
applicationId,
jobRequest.MaxApplicationSubmissions,
jobResources);
var submittedApplication = _yarnRMClient.SubmitApplicationAsync(submissionReq).GetAwaiter().GetResult();
Log.Log(Level.Info, @"Submitted application {0}", submittedApplication.Id);
}
finally
{
if (Directory.Exists(localDriverFolderPath))
{
Directory.Delete(localDriverFolderPath, recursive: true);
}
}
}
public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest)
{
throw new NotSupportedException();
}
public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest)
{
var driverFolder = PrepareDriverFolder(jobRequest);
var submissionJobArgsFilePath = CreateBootstrapAvroJobConfig(jobRequest.JobParameters, driverFolder);
var submissionAppArgsFilePath = CreateBootstrapAvroAppConfig(jobRequest.AppParameters, driverFolder);
_javaClientLauncher.LaunchAsync(JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath)
.LogAndIgnoreExceptionIfAny(Logger, "Java launcher failed");
var fileName = Path.Combine(driverFolder, _fileNames.DriverHttpEndpoint);
JobSubmissionResult result = new LocalJobSubmissionResult(this, fileName);
var msg = string.Format(CultureInfo.CurrentCulture,
"Submitted the Driver for execution. Returned driverUrl is: {0}.", result.DriverUrl);
Logger.Log(Level.Info, msg);
return result;
}
public void Submit(JobRequest jobRequest)
{
var driverFolder = PrepareDriverFolder(jobRequest);
var submissionJobArgsFilePath = CreateBootstrapAvroJobConfig(jobRequest.JobParameters, driverFolder);
var submissionAppArgsFilePath = CreateBootstrapAvroAppConfig(jobRequest.AppParameters, driverFolder);
_javaClientLauncher.LaunchAsync(JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath)
.GetAwaiter().GetResult();
Logger.Log(Level.Info, "Submitted the Driver for execution.");
}
private string PrepareDriverFolder(JobRequest jobRequest)
{
// Prepare the job submission folder
var jobFolder = CreateJobFolder(jobRequest.JobIdentifier);
var driverFolder = Path.Combine(jobFolder, DriverFolderName);
Logger.Log(Level.Info, "Preparing driver folder in " + driverFolder);
_driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, driverFolder);
return driverFolder;
}