/// <summary>
/// Serializes the job parameters to job-submission-params.json.
/// </summary>
internal void SerializeJobFile(JobParameters jobParameters, string localDriverFolderPath, string jobSubmissionDirectory)
{
var serializedArgs = SerializeJobArgsToBytes(jobParameters, localDriverFolderPath, jobSubmissionDirectory);
var submissionJobArgsFilePath = Path.Combine(localDriverFolderPath,
_fileNames.GetJobSubmissionParametersFile());
using (var jobArgsFileStream = new FileStream(submissionJobArgsFilePath, FileMode.CreateNew))
{
jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
}
internal static JobRequest FromJobSubmission(IJobSubmission jobSubmission)
{
return(new JobRequest(
JobParameters.FromJobSubmission(jobSubmission), AppParameters.FromJobSubmission(jobSubmission)));
}
internal JobRequest(JobParameters jobParameters, AppParameters appParameters)
{
_jobParameters = jobParameters;
_appParameters = appParameters;
}
internal JobRequest(JobParameters jobParameters, AppParameters appParameters)
{
_jobParameters = jobParameters;
_appParameters = appParameters;
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string localDriverFolderPath, string jobSubmissionDirectory)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = localDriverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = jobSubmissionDirectory,
dfsJobSubmissionFolder = jobSubmissionDirectory,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
return AvroJsonSerializer<AvroYarnJobSubmissionParameters>.ToBytes(avroYarnJobSubmissionParameters);
}
private SubmitApplication CreateApplicationSubmissionRequest(
JobParameters jobParameters,
string appId,
int maxApplicationSubmissions,
IReadOnlyCollection<JobResource> jobResources)
{
var commandProviderConfigModule = YarnCommandProviderConfiguration.ConfigurationModule;
if (jobParameters.JavaLogLevel == JavaLoggingSetting.Verbose)
{
commandProviderConfigModule = commandProviderConfigModule
.Set(YarnCommandProviderConfiguration.JavaDebugLogging, true.ToString().ToLowerInvariant());
}
if (jobParameters.StdoutFilePath.IsPresent())
{
commandProviderConfigModule = commandProviderConfigModule
.Set(YarnCommandProviderConfiguration.DriverStdoutFilePath, jobParameters.StdoutFilePath.Value);
}
if (jobParameters.StderrFilePath.IsPresent())
{
commandProviderConfigModule = commandProviderConfigModule
.Set(YarnCommandProviderConfiguration.DriverStderrFilePath, jobParameters.StderrFilePath.Value);
}
var yarnJobCommandProvider = _injector.ForkInjector(commandProviderConfigModule.Build())
.GetInstance<IYarnJobCommandProvider>();
var command = yarnJobCommandProvider.GetJobSubmissionCommand();
Log.Log(Level.Verbose, "Command for YARN: {0}", command);
Log.Log(Level.Verbose, "ApplicationID: {0}", appId);
Log.Log(Level.Verbose, "MaxApplicationSubmissions: {0}", maxApplicationSubmissions);
foreach (var jobResource in jobResources)
{
Log.Log(Level.Verbose, "Remote file: {0}", jobResource.RemoteUploadPath);
}
var submitApplication = new SubmitApplication
{
ApplicationId = appId,
ApplicationName = jobParameters.JobIdentifier,
AmResource = new Resouce
{
MemoryMB = jobParameters.DriverMemoryInMB,
VCores = 1 // keeping parity with existing code
},
MaxAppAttempts = maxApplicationSubmissions,
ApplicationType = REEFApplicationType,
KeepContainersAcrossApplicationAttempts = true,
Queue = @"default", // keeping parity with existing code
Priority = 1, // keeping parity with existing code
UnmanagedAM = false,
AmContainerSpec = new AmContainerSpec
{
LocalResources = CreateLocalResources(jobResources),
Commands = new Commands
{
Command = command
}
}
};
return submitApplication;
}
private string CreateBootstrapAvroJobConfig(JobParameters jobParameters, string driverFolder)
{
var bootstrapJobArgs = new AvroJobSubmissionParameters
{
jobSubmissionFolder = driverFolder,
jobId = jobParameters.JobIdentifier,
};
var bootstrapLocalJobArgs = new AvroLocalJobSubmissionParameters
{
sharedJobSubmissionParameters = bootstrapJobArgs,
driverStdoutFilePath = jobParameters.StdoutFilePath.IsPresent() ? jobParameters.StdoutFilePath.Value : _fileNames.GetDriverStdoutFileName(),
driverStderrFilePath = jobParameters.StderrFilePath.IsPresent() ? jobParameters.StderrFilePath.Value : _fileNames.GetDriverStderrFileName()
};
var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer<AvroLocalJobSubmissionParameters>.ToBytes(bootstrapLocalJobArgs);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return submissionArgsFilePath;
}
/// <summary>
/// Serializes the job parameters to job-submission-params.json.
/// </summary>
internal string SerializeJobFile(JobParameters jobParameters, IInjector paramInjector, string driverFolderPath)
{
var serializedArgs = SerializeJobArgsToBytes(jobParameters, driverFolderPath);
var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return submissionArgsFilePath;
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
driverMemory = jobParameters.DriverMemoryInMB,
maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions,
driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value,
driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value
};
return AvroJsonSerializer<AvroYarnClusterJobSubmissionParameters>.ToBytes(avroYarnClusterJobSubmissionParameters);
}
