internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
sharedJobSubmissionParameters = avroJobSubmissionParameters,
fileSystemUrl = _fileSystemUrl,
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
// TODO[JIRA REEF-1887] Deprecated. Remove in REEF 0.18.
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
driverMemory = jobParameters.DriverMemoryInMB,
environmentVariablesMap = jobParameters.JobSubmissionEnvMap,
maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions,
driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value,
driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value
};
return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters));
}
private string CreateBootstrapAvroJobConfig(JobParameters jobParameters, string driverFolder)
{
var bootstrapJobArgs = new AvroJobSubmissionParameters
{
jobSubmissionFolder = driverFolder,
jobId = jobParameters.JobIdentifier,
};
var bootstrapLocalJobArgs = new AvroLocalJobSubmissionParameters
{
sharedJobSubmissionParameters = bootstrapJobArgs,
driverStdoutFilePath = jobParameters.StdoutFilePath.IsPresent() ? jobParameters.StdoutFilePath.Value : _fileNames.GetDriverStdoutFileName(),
driverStderrFilePath = jobParameters.StderrFilePath.IsPresent() ? jobParameters.StderrFilePath.Value : _fileNames.GetDriverStderrFileName()
};
var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroLocalJobSubmissionParameters> .ToBytes(bootstrapLocalJobArgs);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return(submissionArgsFilePath);
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
driverMemory = jobParameters.DriverMemoryInMB,
maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions,
driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value,
driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value
};
return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters));
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
driverMemory = jobParameters.DriverMemoryInMB,
maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions
};
return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters));
}
private string CreateBootstrapAvroConfig(IJobSubmission jobSubmission, string driverFolder)
{
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
var bootstrapArgs = new AvroJobSubmissionParameters
{
jobSubmissionFolder = driverFolder,
jobId = jobSubmission.JobIdentifier,
tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(),
};
var avroLocalBootstrapArgs = new AvroLocalJobSubmissionParameters
{
sharedJobSubmissionParameters = bootstrapArgs,
maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators
};
var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroLocalJobSubmissionParameters> .ToBytes(avroLocalBootstrapArgs);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return(submissionArgsFilePath);
}
/// <summary>
/// Initializes a new instance of the <see cref="AvroYarnJobSubmissionParameters"/> class.
/// </summary>
/// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param>
/// <param name="dfsJobSubmissionFolder">The dfsJobSubmissionFolder.</param>
/// <param name="jobSubmissionDirectoryPrefix">The jobSubmissionDirectoryPrefix.</param>
/// <param name="fileSystemUrl">The file system URL prefix.</param>
public AvroYarnJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, string dfsJobSubmissionFolder, string fileSystemUrl, string jobSubmissionDirectoryPrefix)
{
this.sharedJobSubmissionParameters = sharedJobSubmissionParameters;
this.dfsJobSubmissionFolder = dfsJobSubmissionFolder;
this.fileSystemUrl = fileSystemUrl;
this.jobSubmissionDirectoryPrefix = jobSubmissionDirectoryPrefix;
}
/// <summary>
/// Initializes a new instance of the <see cref="AvroYarnJobSubmissionParameters"/> class.
/// </summary>
/// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param>
/// <param name="driverMemory">The driverMemory.</param>
/// <param name="driverRecoveryTimeout">The driverRecoveryTimeout.</param>
/// <param name="dfsJobSubmissionFolder">The dfsJobSubmissionFolder.</param>
/// <param name="jobSubmissionDirectoryPrefix">The jobSubmissionDirectoryPrefix.</param>
public AvroYarnJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, int driverMemory, int driverRecoveryTimeout, string dfsJobSubmissionFolder, string jobSubmissionDirectoryPrefix)
{
this.sharedJobSubmissionParameters = sharedJobSubmissionParameters;
this.driverMemory = driverMemory;
this.driverRecoveryTimeout = driverRecoveryTimeout;
this.dfsJobSubmissionFolder = dfsJobSubmissionFolder;
this.jobSubmissionDirectoryPrefix = jobSubmissionDirectoryPrefix;
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string localDriverFolderPath, string jobSubmissionDirectory)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = localDriverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = jobSubmissionDirectory,
dfsJobSubmissionFolder = jobSubmissionDirectory,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
return(AvroJsonSerializer <AvroYarnJobSubmissionParameters> .ToBytes(avroYarnJobSubmissionParameters));
}
private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
{
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);
// TODO: Remove this when we have a generalized way to pass config to java
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(),
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
driverMemory = jobSubmission.DriverMemory,
driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
maxApplicationSubmissions = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(),
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters
};
var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
// Submit the driver
_javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
}
/// <summary>
/// Initializes a new instance of the <see cref="AvroYarnJobSubmissionParameters"/> class.
/// </summary>
/// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param>
/// <param name="driverMemory">The driverMemory.</param>
/// <param name="driverRecoveryTimeout">The driverRecoveryTimeout.</param>
/// <param name="dfsJobSubmissionFolder">The dfsJobSubmissionFolder.</param>
/// <param name="jobSubmissionDirectoryPrefix">The jobSubmissionDirectoryPrefix.</param>
public AvroYarnJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, int driverMemory, int driverRecoveryTimeout, string dfsJobSubmissionFolder, string jobSubmissionDirectoryPrefix)
{
this.sharedJobSubmissionParameters = sharedJobSubmissionParameters;
this.driverMemory = driverMemory;
this.driverRecoveryTimeout = driverRecoveryTimeout;
this.dfsJobSubmissionFolder = dfsJobSubmissionFolder;
this.jobSubmissionDirectoryPrefix = jobSubmissionDirectoryPrefix;
}
/// <summary>
/// Initializes a new instance of the <see cref="AvroLocalJobSubmissionParameters"/> class.
/// </summary>
/// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param>
/// <param name="maxNumberOfConcurrentEvaluators">The maxNumberOfConcurrentEvaluators.</param>
public AvroLocalJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, int maxNumberOfConcurrentEvaluators)
{
this.sharedJobSubmissionParameters = sharedJobSubmissionParameters;
this.maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators;
}
/// <summary>
/// Initializes a new instance of the <see cref="AvroYarnJobSubmissionParameters"/> class.
/// </summary>
/// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param>
/// <param name="dfsJobSubmissionFolder">The dfsJobSubmissionFolder.</param>
/// <param name="jobSubmissionDirectoryPrefix">The jobSubmissionDirectoryPrefix.</param>
public AvroYarnJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, string dfsJobSubmissionFolder, string jobSubmissionDirectoryPrefix)
{
this.sharedJobSubmissionParameters = sharedJobSubmissionParameters;
this.dfsJobSubmissionFolder = dfsJobSubmissionFolder;
this.jobSubmissionDirectoryPrefix = jobSubmissionDirectoryPrefix;
}
/// <summary>
/// Initializes a new instance of the <see cref="AvroLocalJobSubmissionParameters"/> class.
/// </summary>
/// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param>
/// <param name="maxNumberOfConcurrentEvaluators">The maxNumberOfConcurrentEvaluators.</param>
public AvroLocalJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, int maxNumberOfConcurrentEvaluators)
{
this.sharedJobSubmissionParameters = sharedJobSubmissionParameters;
this.maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators;
}
/// <summary>
/// Initializes a new instance of the <see cref="AvroLocalJobSubmissionParameters"/> class.
/// </summary>
/// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param>
/// <param name="driverStdoutFilePath">The driverStdoutFilePath.</param>
/// <param name="driverStderrFilePath">The driverStderrFilePath.</param>
public AvroLocalJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, string driverStdoutFilePath, string driverStderrFilePath)
{
this.sharedJobSubmissionParameters = sharedJobSubmissionParameters;
this.driverStdoutFilePath = driverStdoutFilePath;
this.driverStderrFilePath = driverStderrFilePath;
}
/// <summary>
/// Initializes a new instance of the <see cref="AvroLocalJobSubmissionParameters"/> class.
/// </summary>
/// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param>
/// <param name="driverStdoutFilePath">The driverStdoutFilePath.</param>
/// <param name="driverStderrFilePath">The driverStderrFilePath.</param>
public AvroLocalJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, string driverStdoutFilePath, string driverStderrFilePath)
{
this.sharedJobSubmissionParameters = sharedJobSubmissionParameters;
this.driverStdoutFilePath = driverStdoutFilePath;
this.driverStderrFilePath = driverStderrFilePath;
}
