public JobResource UploadJobResource(string driverLocalFolderPath)
{
driverLocalFolderPath = driverLocalFolderPath.TrimEnd('\\') + @"\";
var driverUploadPath = _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().TrimEnd('/') + @"/";
Log.Log(Level.Verbose, "DriverFolderPath: {0} DriverUploadPath: {1}", driverLocalFolderPath, driverUploadPath);
var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath);
var destinationPath = driverUploadPath + Path.GetFileName(archivePath);
var remoteFileUri = _fileSystem.CreateUriForPath(destinationPath);
Log.Log(Level.Verbose, @"Copy {0} to {1}", archivePath, remoteFileUri);
var parentDirectoryUri = _fileSystem.CreateUriForPath(driverUploadPath);
_fileSystem.CreateDirectory(parentDirectoryUri);
_fileSystem.CopyFromLocal(archivePath, remoteFileUri);
var fileStatus = _fileSystem.GetFileStatus(remoteFileUri);
return(new JobResource
{
LastModificationUnixTimestamp = DateTimeToUnixTimestamp(fileStatus.ModificationTime),
RemoteUploadPath = remoteFileUri.AbsoluteUri,
ResourceSize = fileStatus.LengthBytes
});
}
public JobResource UploadJobResource(string driverLocalFolderPath)
{
driverLocalFolderPath = driverLocalFolderPath.TrimEnd('\\') + @"\";
string driverUploadPath = _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().TrimEnd('/') + @"/";
Log.Log(Level.Info, "DriverFolderPath: {0} DriverUploadPath: {1}", driverLocalFolderPath, driverUploadPath);
var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath);
var resourceDetailsOutputPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"));
_javaLauncher.Launch(JavaClassNameForResourceUploader,
archivePath,
driverUploadPath,
resourceDetailsOutputPath);
return(ParseGeneratedOutputFile(resourceDetailsOutputPath));
}
public FileSystemJobResourceUploader GetJobResourceUploader()
{
var injector = TangFactory.GetTang().NewInjector();
JobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().Returns(AnyDriverResourceUploadPath);
FileSystem.GetFileStatus(new Uri(AnyUploadedResourceAbsoluteUri))
.Returns(new FileStatus(Epoch + TimeSpan.FromSeconds(AnyModificationTime), AnyResourceSize));
ResourceArchiveFileGenerator.CreateArchiveToUpload(AnyDriverLocalFolderPath)
.Returns(AnyLocalArchivePath);
FileSystem.CreateUriForPath(AnyDriverResourceUploadPath)
.Returns(new Uri(AnyScheme + AnyHost + AnyDriverResourceUploadPath));
FileSystem.CreateUriForPath(AnyUploadedResourcePath)
.Returns(new Uri(AnyUploadedResourceAbsoluteUri));
injector.BindVolatileInstance(GenericType <IJobSubmissionDirectoryProvider> .Class, JobSubmissionDirectoryProvider);
injector.BindVolatileInstance(GenericType <IResourceArchiveFileGenerator> .Class, ResourceArchiveFileGenerator);
injector.BindVolatileInstance(GenericType <IFileSystem> .Class, FileSystem);
return(injector.GetInstance <FileSystemJobResourceUploader>());
}
public void Submit(JobRequest jobRequest)
{
string jobId = jobRequest.JobIdentifier;
// todo: Future client interface should be async.
// Using GetAwaiter().GetResult() instead of .Result to avoid exception
// getting wrapped in AggregateException.
var newApplication = _yarnRMClient.CreateNewApplicationAsync().GetAwaiter().GetResult();
string applicationId = newApplication.ApplicationId;
// create job submission remote path
string jobSubmissionDirectory =
_jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory(applicationId);
// create local driver folder.
var localDriverFolderPath = CreateDriverFolder(jobId, applicationId);
try
{
Log.Log(Level.Verbose, "Preparing driver folder in {0}", localDriverFolderPath);
_driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, localDriverFolderPath);
// prepare configuration
var paramInjector = TangFactory.GetTang().NewInjector(jobRequest.DriverConfigurations.ToArray());
_paramSerializer.SerializeAppFile(jobRequest.AppParameters, paramInjector, localDriverFolderPath);
_paramSerializer.SerializeJobFile(jobRequest.JobParameters, localDriverFolderPath, jobSubmissionDirectory);
var archiveResource =
_jobResourceUploader.UploadArchiveResourceAsync(localDriverFolderPath, jobSubmissionDirectory)
.GetAwaiter()
.GetResult();
// Path to the job args file.
var jobArgsFilePath = Path.Combine(localDriverFolderPath, _fileNames.GetJobSubmissionParametersFile());
var argFileResource =
_jobResourceUploader.UploadFileResourceAsync(jobArgsFilePath, jobSubmissionDirectory)
.GetAwaiter()
.GetResult();
// upload prepared folder to DFS
var jobResources = new List <JobResource> {
archiveResource, argFileResource
};
// submit job
Log.Log(Level.Verbose, @"Assigned application id {0}", applicationId);
var submissionReq = CreateApplicationSubmissionRequest(
jobRequest.JobParameters,
applicationId,
jobRequest.MaxApplicationSubmissions,
jobResources);
var submittedApplication = _yarnRMClient.SubmitApplicationAsync(submissionReq).GetAwaiter().GetResult();
Log.Log(Level.Info, @"Submitted application {0}", submittedApplication.Id);
}
finally
{
if (Directory.Exists(localDriverFolderPath))
{
Directory.Delete(localDriverFolderPath, recursive: true);
}
}
}