internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { jobSubmissionDirectoryPrefix = _jobSubmissionPrefix, sharedJobSubmissionParameters = avroJobSubmissionParameters }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, driverMemory = jobParameters.DriverMemoryInMB, maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions, driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ? _fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value, driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ? _fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value }; return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters)); }
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { jobSubmissionDirectoryPrefix = _jobSubmissionPrefix, sharedJobSubmissionParameters = avroJobSubmissionParameters }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, driverMemory = jobParameters.DriverMemoryInMB, maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions }; return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters)); }
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { sharedJobSubmissionParameters = avroJobSubmissionParameters, fileSystemUrl = _fileSystemUrl, jobSubmissionDirectoryPrefix = _jobSubmissionPrefix }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { // TODO[JIRA REEF-1887] Deprecated. Remove in REEF 0.18. securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, driverMemory = jobParameters.DriverMemoryInMB, environmentVariablesMap = jobParameters.JobSubmissionEnvMap, maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions, driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ? _fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value, driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ? _fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value }; return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters)); }
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string localDriverFolderPath, string jobSubmissionDirectory) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = localDriverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { jobSubmissionDirectoryPrefix = jobSubmissionDirectory, dfsJobSubmissionFolder = jobSubmissionDirectory, sharedJobSubmissionParameters = avroJobSubmissionParameters }; return(AvroJsonSerializer <AvroYarnJobSubmissionParameters> .ToBytes(avroYarnJobSubmissionParameters)); }
private void Launch(IJobSubmission jobSubmission, string driverFolderPath) { _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath); // TODO: Remove this when we have a generalized way to pass config to java var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray()); var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobSubmission.JobIdentifier, tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(), tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(), tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(), jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { driverMemory = jobSubmission.DriverMemory, driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(), jobSubmissionDirectoryPrefix = _jobSubmissionPrefix, sharedJobSubmissionParameters = avroJobSubmissionParameters }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { maxApplicationSubmissions = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(), securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters }; var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { var serializedArgs = AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters); argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } // Submit the driver _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath); Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier); }