public void Submit(IJobSubmission jobSubmission) { var driverFolder = PrepareDriverFolder(jobSubmission); var submissionArgsFilePath = CreateBootstrapAvroConfig(jobSubmission, driverFolder); _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath); Logger.Log(Level.Info, "Submitted the Driver for execution."); }
private void Launch(IJobSubmission jobSubmission, string driverFolderPath) { _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath); // TODO: Remove this when we have a generalized way to pass config to java var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray()); var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobSubmission.JobIdentifier, tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(), tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(), tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(), jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { driverMemory = jobSubmission.DriverMemory, driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(), jobSubmissionDirectoryPrefix = _jobSubmissionPrefix, sharedJobSubmissionParameters = avroJobSubmissionParameters }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { maxApplicationSubmissions = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(), securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters }; var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { var serializedArgs = AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters); argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } // Submit the driver _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath); Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier); }
public JobResource UploadJobResource(string driverLocalFolderPath) { driverLocalFolderPath = driverLocalFolderPath.TrimEnd('\\') + @"\"; string driverUploadPath = _jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory().TrimEnd('/') + @"/"; Log.Log(Level.Info, "DriverFolderPath: {0} DriverUploadPath: {1}", driverLocalFolderPath, driverUploadPath); var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath); var resourceDetailsOutputPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); _javaLauncher.Launch(JavaClassNameForResourceUploader, archivePath, driverUploadPath, resourceDetailsOutputPath); return(ParseGeneratedOutputFile(resourceDetailsOutputPath)); }