/// <summary>
/// Initializes a new instance of the <see cref="AvroYarnClusterJobSubmissionParameters"/> class.
/// </summary>
/// <param name="yarnJobSubmissionParameters">The yarnJobSubmissionParameters.</param>
/// <param name="maxApplicationSubmissions">The maxApplicationSubmissions.</param>
/// <param name="securityTokenKind">The securityTokenKind.</param>
/// <param name="securityTokenService">The securityTokenService.</param>
public AvroYarnClusterJobSubmissionParameters(AvroYarnJobSubmissionParameters yarnJobSubmissionParameters, int maxApplicationSubmissions, string securityTokenKind, string securityTokenService)
{
this.yarnJobSubmissionParameters = yarnJobSubmissionParameters;
this.maxApplicationSubmissions = maxApplicationSubmissions;
this.securityTokenKind = securityTokenKind;
this.securityTokenService = securityTokenService;
}
/// <summary>
/// Initializes a new instance of the <see cref="AvroYarnClusterJobSubmissionParameters"/> class.
/// </summary>
/// <param name="yarnJobSubmissionParameters">The yarnJobSubmissionParameters.</param>
/// <param name="maxApplicationSubmissions">The maxApplicationSubmissions.</param>
/// <param name="securityTokenKind">The securityTokenKind.</param>
/// <param name="securityTokenService">The securityTokenService.</param>
public AvroYarnClusterJobSubmissionParameters(AvroYarnJobSubmissionParameters yarnJobSubmissionParameters, int maxApplicationSubmissions, string securityTokenKind, string securityTokenService)
{
this.yarnJobSubmissionParameters = yarnJobSubmissionParameters;
this.maxApplicationSubmissions = maxApplicationSubmissions;
this.securityTokenKind = securityTokenKind;
this.securityTokenService = securityTokenService;
}
/// <summary>
/// Initializes a new instance of the <see cref="AvroYarnClusterJobSubmissionParameters"/> class.
/// </summary>
/// <param name="yarnJobSubmissionParameters">The yarnJobSubmissionParameters.</param>
/// <param name="securityTokenKind">The securityTokenKind.</param>
/// <param name="securityTokenService">The securityTokenService.</param>
/// <param name="driverMemory">The driverMemory.</param>
/// <param name="maxApplicationSubmissions">The maxApplicationSubmissions.</param>
/// <param name="driverStdoutFilePath">The driverStdoutFilePath.</param>
/// <param name="driverStderrFilePath">The driverStderrFilePath.</param>
public AvroYarnClusterJobSubmissionParameters(AvroYarnJobSubmissionParameters yarnJobSubmissionParameters, string securityTokenKind, string securityTokenService, int driverMemory, int maxApplicationSubmissions, string driverStdoutFilePath, string driverStderrFilePath)
{
this.yarnJobSubmissionParameters = yarnJobSubmissionParameters;
this.securityTokenKind = securityTokenKind;
this.securityTokenService = securityTokenService;
this.driverMemory = driverMemory;
this.maxApplicationSubmissions = maxApplicationSubmissions;
this.driverStdoutFilePath = driverStdoutFilePath;
this.driverStderrFilePath = driverStderrFilePath;
}
/// <summary>
/// Initializes a new instance of the <see cref="AvroYarnClusterJobSubmissionParameters"/> class.
/// </summary>
/// <param name="yarnJobSubmissionParameters">The yarnJobSubmissionParameters.</param>
/// <param name="securityTokenKind">The securityTokenKind.</param>
/// <param name="securityTokenService">The securityTokenService.</param>
/// <param name="driverMemory">The driverMemory.</param>
/// <param name="envMap">The envMap.</param>
/// <param name="maxApplicationSubmissions">The maxApplicationSubmissions.</param>
/// <param name="driverStdoutFilePath">The driverStdoutFilePath.</param>
/// <param name="driverStderrFilePath">The driverStderrFilePath.</param>
public AvroYarnClusterJobSubmissionParameters(AvroYarnJobSubmissionParameters yarnJobSubmissionParameters, string securityTokenKind, string securityTokenService, int driverMemory, IDictionary <string, string> envMap, int maxApplicationSubmissions, string driverStdoutFilePath, string driverStderrFilePath)
{
this.yarnJobSubmissionParameters = yarnJobSubmissionParameters;
this.securityTokenKind = securityTokenKind;
this.securityTokenService = securityTokenService;
this.driverMemory = driverMemory;
this.environmentVariablesMap = envMap;
this.maxApplicationSubmissions = maxApplicationSubmissions;
this.driverStdoutFilePath = driverStdoutFilePath;
this.driverStderrFilePath = driverStderrFilePath;
}
/// <summary>
/// Initializes a new instance of the <see cref="AvroYarnClusterJobSubmissionParameters"/> class.
/// </summary>
/// <param name="yarnJobSubmissionParameters">The yarnJobSubmissionParameters.</param>
/// <param name="securityTokenKind">The securityTokenKind.</param>
/// <param name="securityTokenService">The securityTokenService.</param>
/// <param name="driverMemory">The driverMemory.</param>
/// <param name="maxApplicationSubmissions">The maxApplicationSubmissions.</param>
/// <param name="driverStdoutFilePath">The driverStdoutFilePath.</param>
/// <param name="driverStderrFilePath">The driverStderrFilePath.</param>
public AvroYarnClusterJobSubmissionParameters(AvroYarnJobSubmissionParameters yarnJobSubmissionParameters, string securityTokenKind, string securityTokenService, int driverMemory, int maxApplicationSubmissions, string driverStdoutFilePath, string driverStderrFilePath)
{
this.yarnJobSubmissionParameters = yarnJobSubmissionParameters;
this.securityTokenKind = securityTokenKind;
this.securityTokenService = securityTokenService;
this.driverMemory = driverMemory;
this.maxApplicationSubmissions = maxApplicationSubmissions;
this.driverStdoutFilePath = driverStdoutFilePath;
this.driverStderrFilePath = driverStderrFilePath;
}
private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
{
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);
// TODO: Remove this when we have a generalized way to pass config to java
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>(),
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
driverMemory = jobSubmission.DriverMemory,
driverRecoveryTimeout = paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
maxApplicationSubmissions = paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(),
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters
};
var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer<AvroYarnClusterJobSubmissionParameters>.ToBytes(avroYarnClusterJobSubmissionParameters);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
// Submit the driver
_javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
}
internal byte[] SerializeJobArgsToBytes(IJobSubmission jobSubmission, string localDriverFolderPath, string jobSubmissionDirectory)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
jobSubmissionFolder = localDriverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = jobSubmissionDirectory,
dfsJobSubmissionFolder = jobSubmissionDirectory,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
return AvroJsonSerializer<AvroYarnJobSubmissionParameters>.ToBytes(avroYarnJobSubmissionParameters);
}
public void Submit(IJobSubmission jobSubmission)
{
string jobId = jobSubmission.JobIdentifier;
// todo: Future client interface should be async.
// Using GetAwaiter().GetResult() instead of .Result to avoid exception
// getting wrapped in AggregateException.
var newApplication = _yarnRMClient.CreateNewApplicationAsync().GetAwaiter().GetResult();
string applicationId = newApplication.ApplicationId;
// create job submission remote path
string jobSubmissionDirectory =
_jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory(applicationId);
// create local driver folder.
var localDriverFolderPath = CreateDriverFolder(jobId, applicationId);
try
{
Log.Log(Level.Verbose, "Preparing driver folder in {0}", localDriverFolderPath);
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, localDriverFolderPath);
// prepare configuration
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
int maxApplicationSubmissions =
paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>();
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>(),
jobSubmissionFolder = localDriverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
driverMemory = jobSubmission.DriverMemory,
driverRecoveryTimeout =
paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
jobSubmissionDirectoryPrefix = jobSubmissionDirectory,
dfsJobSubmissionFolder = jobSubmissionDirectory,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var submissionArgsFilePath = Path.Combine(localDriverFolderPath,
_fileNames.GetLocalFolderPath(),
_fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs =
AvroJsonSerializer<AvroYarnJobSubmissionParameters>.ToBytes(avroYarnJobSubmissionParameters);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
// upload prepared folder to DFS
var jobResource = _jobResourceUploader.UploadJobResource(localDriverFolderPath, jobSubmissionDirectory);
// submit job
Log.Log(Level.Verbose, @"Assigned application id {0}", applicationId);
var submissionReq = CreateApplicationSubmissionRequest(jobSubmission,
applicationId,
maxApplicationSubmissions,
jobResource);
var submittedApplication = _yarnRMClient.SubmitApplicationAsync(submissionReq).GetAwaiter().GetResult();
Log.Log(Level.Info, @"Submitted application {0}", submittedApplication.Id);
}
finally
{
if (Directory.Exists(localDriverFolderPath))
{
Directory.Delete(localDriverFolderPath, recursive: true);
}
}
}
internal byte[] SerializeJobArgsToBytes(IJobSubmission jobSubmission, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters
};
return AvroJsonSerializer<AvroYarnClusterJobSubmissionParameters>.ToBytes(avroYarnClusterJobSubmissionParameters);
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
driverMemory = jobParameters.DriverMemoryInMB,
maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions,
driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value,
driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value
};
return AvroJsonSerializer<AvroYarnClusterJobSubmissionParameters>.ToBytes(avroYarnClusterJobSubmissionParameters);
}