internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
        {
            var avroJobSubmissionParameters = new AvroJobSubmissionParameters
            {
                jobId = jobParameters.JobIdentifier,
                jobSubmissionFolder = driverFolderPath
            };

            var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
            {
                jobSubmissionDirectoryPrefix  = _jobSubmissionPrefix,
                sharedJobSubmissionParameters = avroJobSubmissionParameters
            };

            var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
            {
                securityTokenKind           = _securityTokenKind,
                securityTokenService        = _securityTokenService,
                yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
                driverMemory = jobParameters.DriverMemoryInMB,
                maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions,
                driverStdoutFilePath      = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ?
                                            _fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value,
                driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ?
                                       _fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value
            };

            return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters));
        }
Ejemplo n.º 2
0
        internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
        {
            var avroJobSubmissionParameters = new AvroJobSubmissionParameters
            {
                jobId = jobParameters.JobIdentifier,
                jobSubmissionFolder = driverFolderPath
            };

            var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
            {
                sharedJobSubmissionParameters = avroJobSubmissionParameters,
                fileSystemUrl = _fileSystemUrl,
                jobSubmissionDirectoryPrefix = _jobSubmissionPrefix
            };

            var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
            {
                // TODO[JIRA REEF-1887] Deprecated. Remove in REEF 0.18.
                securityTokenKind    = _securityTokenKind,
                securityTokenService = _securityTokenService,

                yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
                driverMemory              = jobParameters.DriverMemoryInMB,
                environmentVariablesMap   = jobParameters.JobSubmissionEnvMap,
                maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions,
                driverStdoutFilePath      = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ?
                                            _fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value,
                driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ?
                                       _fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value
            };

            return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters));
        }