private string PrepareDriverFolder(IJobSubmission jobSubmission) { // Prepare the job submission folder var jobFolder = CreateJobFolder(jobSubmission.JobIdentifier); var driverFolder = Path.Combine(jobFolder, DriverFolderName); Logger.Log(Level.Info, "Preparing driver folder in " + driverFolder); _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolder); return(driverFolder); }
private string PrepareDriverFolder(JobRequest jobRequest) { // Prepare the job submission folder var jobFolder = CreateJobFolder(jobRequest.JobIdentifier); var driverFolder = Path.Combine(jobFolder, DriverFolderName); Logger.Log(Level.Verbose, "Preparing driver folder in " + driverFolder); _driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, driverFolder); return(driverFolder); }
private void Launch(JobRequest jobRequest, string driverFolderPath) { _driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, driverFolderPath); // TODO: Remove this when we have a generalized way to pass config to java var paramInjector = TangFactory.GetTang().NewInjector(jobRequest.DriverConfigurations.ToArray()); var submissionJobArgsFilePath = _paramSerializer.SerializeJobFile(jobRequest.JobParameters, paramInjector, driverFolderPath); var submissionAppArgsFilePath = _paramSerializer.SerializeAppFile(jobRequest.AppParameters, paramInjector, driverFolderPath); // Submit the driver _javaClientLauncher.LaunchAsync(JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath) .GetAwaiter() .GetResult(); Logger.Log(Level.Info, "Submitted the Driver for execution." + jobRequest.JobIdentifier); }
/// <summary> /// Creates a JAR file for the job submission. /// </summary> /// <param name="jobRequest">Job request received from the client code.</param> /// <param name="azureBatchjobId">Azure Batch job Id going to be launched.</param> /// <returns>A string path to file.</returns> public string CreateJobSubmissionJAR(JobRequest jobRequest, string azureBatchjobId) { _avroAzureBatchJobSubmissionParameters.sharedJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobRequest.JobIdentifier, //// This is dummy in Azure Batch, as it does not use jobSubmissionFolder in Azure Batch. jobSubmissionFolder = Path.PathSeparator.ToString() }; string localDriverFolderPath = CreateDriverFolder(azureBatchjobId); _driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, localDriverFolderPath); SerializeJobFile(localDriverFolderPath, _avroAzureBatchJobSubmissionParameters); return(_resourceArchiveFileGenerator.CreateArchiveToUpload(localDriverFolderPath)); }
private void Launch(IJobSubmission jobSubmission, string driverFolderPath) { _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath); // TODO: Remove this when we have a generalized way to pass config to java var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray()); var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobSubmission.JobIdentifier, tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(), tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(), tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(), jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { driverMemory = jobSubmission.DriverMemory, driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(), jobSubmissionDirectoryPrefix = _jobSubmissionPrefix, sharedJobSubmissionParameters = avroJobSubmissionParameters }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { maxApplicationSubmissions = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(), securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters }; var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { var serializedArgs = AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters); argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } // Submit the driver _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath); Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier); }
public void Submit(IJobSubmission jobSubmission) { // Prepare the job submission folder var jobFolder = CreateJobFolder(jobSubmission.JobIdentifier); var driverFolder = Path.Combine(jobFolder, DriverFolderName); Logger.Log(Level.Info, "Preparing driver folder in " + driverFolder); _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolder); //TODO: Remove this when we have a generalized way to pass config to java var javaParams = TangFactory.GetTang() .NewInjector(jobSubmission.DriverConfigurations.ToArray()) .GetInstance <ClrClient2JavaClientCuratedParameters>(); _javaClientLauncher.Launch(JavaClassName, driverFolder, jobSubmission.JobIdentifier, _numberOfEvaluators.ToString(), javaParams.TcpPortRangeStart.ToString(), javaParams.TcpPortRangeCount.ToString(), javaParams.TcpPortRangeTryCount.ToString() ); Logger.Log(Level.Info, "Submitted the Driver for execution."); }
public void Submit(JobRequest jobRequest) { string jobId = jobRequest.JobIdentifier; // todo: Future client interface should be async. // Using GetAwaiter().GetResult() instead of .Result to avoid exception // getting wrapped in AggregateException. var newApplication = _yarnRMClient.CreateNewApplicationAsync().GetAwaiter().GetResult(); string applicationId = newApplication.ApplicationId; // create job submission remote path string jobSubmissionDirectory = _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory(applicationId); // create local driver folder. var localDriverFolderPath = CreateDriverFolder(jobId, applicationId); try { Log.Log(Level.Verbose, "Preparing driver folder in {0}", localDriverFolderPath); _driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, localDriverFolderPath); // prepare configuration var paramInjector = TangFactory.GetTang().NewInjector(jobRequest.DriverConfigurations.ToArray()); _paramSerializer.SerializeAppFile(jobRequest.AppParameters, paramInjector, localDriverFolderPath); _paramSerializer.SerializeJobFile(jobRequest.JobParameters, localDriverFolderPath, jobSubmissionDirectory); var archiveResource = _jobResourceUploader.UploadArchiveResourceAsync(localDriverFolderPath, jobSubmissionDirectory) .GetAwaiter() .GetResult(); // Path to the job args file. var jobArgsFilePath = Path.Combine(localDriverFolderPath, _fileNames.GetJobSubmissionParametersFile()); var argFileResource = _jobResourceUploader.UploadFileResourceAsync(jobArgsFilePath, jobSubmissionDirectory) .GetAwaiter() .GetResult(); // upload prepared folder to DFS var jobResources = new List <JobResource> { archiveResource, argFileResource }; // submit job Log.Log(Level.Verbose, @"Assigned application id {0}", applicationId); var submissionReq = CreateApplicationSubmissionRequest( jobRequest.JobParameters, applicationId, jobRequest.MaxApplicationSubmissions, jobResources); var submittedApplication = _yarnRMClient.SubmitApplicationAsync(submissionReq).GetAwaiter().GetResult(); Log.Log(Level.Info, @"Submitted application {0}", submittedApplication.Id); } finally { if (Directory.Exists(localDriverFolderPath)) { Directory.Delete(localDriverFolderPath, recursive: true); } } }