/// <summary> /// Serializes the job parameters to job-submission-params.json. /// </summary> internal void SerializeJobFile(JobParameters jobParameters, string localDriverFolderPath, string jobSubmissionDirectory) { var serializedArgs = SerializeJobArgsToBytes(jobParameters, localDriverFolderPath, jobSubmissionDirectory); var submissionJobArgsFilePath = Path.Combine(localDriverFolderPath, _fileNames.GetJobSubmissionParametersFile()); using (var jobArgsFileStream = new FileStream(submissionJobArgsFilePath, FileMode.CreateNew)) { jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } }
internal static JobRequest FromJobSubmission(IJobSubmission jobSubmission) { return(new JobRequest( JobParameters.FromJobSubmission(jobSubmission), AppParameters.FromJobSubmission(jobSubmission))); }
internal JobRequest(JobParameters jobParameters, AppParameters appParameters) { _jobParameters = jobParameters; _appParameters = appParameters; }
internal JobRequest(JobParameters jobParameters, AppParameters appParameters) { _jobParameters = jobParameters; _appParameters = appParameters; }
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string localDriverFolderPath, string jobSubmissionDirectory) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = localDriverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { jobSubmissionDirectoryPrefix = jobSubmissionDirectory, dfsJobSubmissionFolder = jobSubmissionDirectory, sharedJobSubmissionParameters = avroJobSubmissionParameters }; return AvroJsonSerializer<AvroYarnJobSubmissionParameters>.ToBytes(avroYarnJobSubmissionParameters); }
private SubmitApplication CreateApplicationSubmissionRequest( JobParameters jobParameters, string appId, int maxApplicationSubmissions, IReadOnlyCollection<JobResource> jobResources) { var commandProviderConfigModule = YarnCommandProviderConfiguration.ConfigurationModule; if (jobParameters.JavaLogLevel == JavaLoggingSetting.Verbose) { commandProviderConfigModule = commandProviderConfigModule .Set(YarnCommandProviderConfiguration.JavaDebugLogging, true.ToString().ToLowerInvariant()); } if (jobParameters.StdoutFilePath.IsPresent()) { commandProviderConfigModule = commandProviderConfigModule .Set(YarnCommandProviderConfiguration.DriverStdoutFilePath, jobParameters.StdoutFilePath.Value); } if (jobParameters.StderrFilePath.IsPresent()) { commandProviderConfigModule = commandProviderConfigModule .Set(YarnCommandProviderConfiguration.DriverStderrFilePath, jobParameters.StderrFilePath.Value); } var yarnJobCommandProvider = _injector.ForkInjector(commandProviderConfigModule.Build()) .GetInstance<IYarnJobCommandProvider>(); var command = yarnJobCommandProvider.GetJobSubmissionCommand(); Log.Log(Level.Verbose, "Command for YARN: {0}", command); Log.Log(Level.Verbose, "ApplicationID: {0}", appId); Log.Log(Level.Verbose, "MaxApplicationSubmissions: {0}", maxApplicationSubmissions); foreach (var jobResource in jobResources) { Log.Log(Level.Verbose, "Remote file: {0}", jobResource.RemoteUploadPath); } var submitApplication = new SubmitApplication { ApplicationId = appId, ApplicationName = jobParameters.JobIdentifier, AmResource = new Resouce { MemoryMB = jobParameters.DriverMemoryInMB, VCores = 1 // keeping parity with existing code }, MaxAppAttempts = maxApplicationSubmissions, ApplicationType = REEFApplicationType, KeepContainersAcrossApplicationAttempts = true, Queue = @"default", // keeping parity with existing code Priority = 1, // keeping parity with existing code UnmanagedAM = false, AmContainerSpec = new AmContainerSpec { LocalResources = CreateLocalResources(jobResources), Commands = new Commands { Command = command } } }; return submitApplication; }
private string CreateBootstrapAvroJobConfig(JobParameters jobParameters, string driverFolder) { var bootstrapJobArgs = new AvroJobSubmissionParameters { jobSubmissionFolder = driverFolder, jobId = jobParameters.JobIdentifier, }; var bootstrapLocalJobArgs = new AvroLocalJobSubmissionParameters { sharedJobSubmissionParameters = bootstrapJobArgs, driverStdoutFilePath = jobParameters.StdoutFilePath.IsPresent() ? jobParameters.StdoutFilePath.Value : _fileNames.GetDriverStdoutFileName(), driverStderrFilePath = jobParameters.StderrFilePath.IsPresent() ? jobParameters.StderrFilePath.Value : _fileNames.GetDriverStderrFileName() }; var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { var serializedArgs = AvroJsonSerializer<AvroLocalJobSubmissionParameters>.ToBytes(bootstrapLocalJobArgs); argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } return submissionArgsFilePath; }
/// <summary> /// Serializes the job parameters to job-submission-params.json. /// </summary> internal string SerializeJobFile(JobParameters jobParameters, IInjector paramInjector, string driverFolderPath) { var serializedArgs = SerializeJobArgsToBytes(jobParameters, driverFolderPath); var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } return submissionArgsFilePath; }
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { jobSubmissionDirectoryPrefix = _jobSubmissionPrefix, sharedJobSubmissionParameters = avroJobSubmissionParameters }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, driverMemory = jobParameters.DriverMemoryInMB, maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions, driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ? _fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value, driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ? _fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value }; return AvroJsonSerializer<AvroYarnClusterJobSubmissionParameters>.ToBytes(avroYarnClusterJobSubmissionParameters); }