public JobResource UploadJobResource(string driverLocalFolderPath) { driverLocalFolderPath = driverLocalFolderPath.TrimEnd('\\') + @"\"; var driverUploadPath = _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().TrimEnd('/') + @"/"; Log.Log(Level.Verbose, "DriverFolderPath: {0} DriverUploadPath: {1}", driverLocalFolderPath, driverUploadPath); var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath); var destinationPath = driverUploadPath + Path.GetFileName(archivePath); var remoteFileUri = _fileSystem.CreateUriForPath(destinationPath); Log.Log(Level.Verbose, @"Copy {0} to {1}", archivePath, remoteFileUri); var parentDirectoryUri = _fileSystem.CreateUriForPath(driverUploadPath); _fileSystem.CreateDirectory(parentDirectoryUri); _fileSystem.CopyFromLocal(archivePath, remoteFileUri); var fileStatus = _fileSystem.GetFileStatus(remoteFileUri); return(new JobResource { LastModificationUnixTimestamp = DateTimeToUnixTimestamp(fileStatus.ModificationTime), RemoteUploadPath = remoteFileUri.AbsoluteUri, ResourceSize = fileStatus.LengthBytes }); }
public JobResource UploadJobResource(string driverLocalFolderPath) { driverLocalFolderPath = driverLocalFolderPath.TrimEnd('\\') + @"\"; string driverUploadPath = _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().TrimEnd('/') + @"/"; Log.Log(Level.Info, "DriverFolderPath: {0} DriverUploadPath: {1}", driverLocalFolderPath, driverUploadPath); var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath); var resourceDetailsOutputPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); _javaLauncher.Launch(JavaClassNameForResourceUploader, archivePath, driverUploadPath, resourceDetailsOutputPath); return(ParseGeneratedOutputFile(resourceDetailsOutputPath)); }
public FileSystemJobResourceUploader GetJobResourceUploader() { var injector = TangFactory.GetTang().NewInjector(); JobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().Returns(AnyDriverResourceUploadPath); FileSystem.GetFileStatus(new Uri(AnyUploadedResourceAbsoluteUri)) .Returns(new FileStatus(Epoch + TimeSpan.FromSeconds(AnyModificationTime), AnyResourceSize)); ResourceArchiveFileGenerator.CreateArchiveToUpload(AnyDriverLocalFolderPath) .Returns(AnyLocalArchivePath); FileSystem.CreateUriForPath(AnyDriverResourceUploadPath) .Returns(new Uri(AnyScheme + AnyHost + AnyDriverResourceUploadPath)); FileSystem.CreateUriForPath(AnyUploadedResourcePath) .Returns(new Uri(AnyUploadedResourceAbsoluteUri)); injector.BindVolatileInstance(GenericType <IJobSubmissionDirectoryProvider> .Class, JobSubmissionDirectoryProvider); injector.BindVolatileInstance(GenericType <IResourceArchiveFileGenerator> .Class, ResourceArchiveFileGenerator); injector.BindVolatileInstance(GenericType <IFileSystem> .Class, FileSystem); return(injector.GetInstance <FileSystemJobResourceUploader>()); }
public void Submit(JobRequest jobRequest) { string jobId = jobRequest.JobIdentifier; // todo: Future client interface should be async. // Using GetAwaiter().GetResult() instead of .Result to avoid exception // getting wrapped in AggregateException. var newApplication = _yarnRMClient.CreateNewApplicationAsync().GetAwaiter().GetResult(); string applicationId = newApplication.ApplicationId; // create job submission remote path string jobSubmissionDirectory = _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory(applicationId); // create local driver folder. var localDriverFolderPath = CreateDriverFolder(jobId, applicationId); try { Log.Log(Level.Verbose, "Preparing driver folder in {0}", localDriverFolderPath); _driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, localDriverFolderPath); // prepare configuration var paramInjector = TangFactory.GetTang().NewInjector(jobRequest.DriverConfigurations.ToArray()); _paramSerializer.SerializeAppFile(jobRequest.AppParameters, paramInjector, localDriverFolderPath); _paramSerializer.SerializeJobFile(jobRequest.JobParameters, localDriverFolderPath, jobSubmissionDirectory); var archiveResource = _jobResourceUploader.UploadArchiveResourceAsync(localDriverFolderPath, jobSubmissionDirectory) .GetAwaiter() .GetResult(); // Path to the job args file. var jobArgsFilePath = Path.Combine(localDriverFolderPath, _fileNames.GetJobSubmissionParametersFile()); var argFileResource = _jobResourceUploader.UploadFileResourceAsync(jobArgsFilePath, jobSubmissionDirectory) .GetAwaiter() .GetResult(); // upload prepared folder to DFS var jobResources = new List <JobResource> { archiveResource, argFileResource }; // submit job Log.Log(Level.Verbose, @"Assigned application id {0}", applicationId); var submissionReq = CreateApplicationSubmissionRequest( jobRequest.JobParameters, applicationId, jobRequest.MaxApplicationSubmissions, jobResources); var submittedApplication = _yarnRMClient.SubmitApplicationAsync(submissionReq).GetAwaiter().GetResult(); Log.Log(Level.Info, @"Submitted application {0}", submittedApplication.Id); } finally { if (Directory.Exists(localDriverFolderPath)) { Directory.Delete(localDriverFolderPath, recursive: true); } } }