private string CreateBootstrapAvroAppConfig(AppParameters appParameters, string driverFolder)
{
var paramInjector = TangFactory.GetTang().NewInjector(appParameters.DriverConfigurations.ToArray());
var bootstrapAppArgs = new AvroAppSubmissionParameters
{
tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(),
};
var avroLocalBootstrapAppArgs = new AvroLocalAppSubmissionParameters
{
sharedAppSubmissionParameters = bootstrapAppArgs,
maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators
};
var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetAppSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroLocalAppSubmissionParameters> .ToBytes(avroLocalBootstrapAppArgs);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return(submissionArgsFilePath);
}
/// <summary>
/// Builds the command to be submitted to YARNRM
/// </summary>
/// <returns>Command string</returns>
public string GetJobSubmissionCommand()
{
var sb = new StringBuilder();
sb.Append(_fileNames.GetBridgeExePath());
sb.Append(" " + JavaExe);
sb.Append(" " + JvmOptionsPermSize);
sb.Append(" " + string.Format(JvmOptionsMaxPermSizeFormat, _driverMaxPermSizeMB));
sb.Append(" " +
string.Format(JvmOptionsMaxMemoryAllocationPoolSizeFormat, _driverMaxMemoryAllocationPoolSizeMB));
sb.Append(" " + ClassPathToken);
var yarnClasspathList = new List <string>(_yarnCommandLineEnvironment.GetYarnClasspathList())
{
string.Format("{0}/{1}/*", _fileNames.GetReefFolderName(), _fileNames.GetLocalFolderName()),
string.Format("{0}/{1}/*", _fileNames.GetReefFolderName(), _fileNames.GetGlobalFolderName())
};
sb.Append(" " + string.Join(";", yarnClasspathList));
sb.Append(" " + ProcReefProperty);
if (_enableDebugLogging)
{
sb.Append(" " + JavaLoggingProperty);
}
sb.Append(" " + LauncherClassName);
sb.Append(" " + _fileNames.GetJobSubmissionParametersFile());
sb.Append(" " +
string.Format("{0}/{1}/{2}",
_fileNames.GetReefFolderName(),
_fileNames.GetLocalFolderName(),
_fileNames.GetAppSubmissionParametersFile()));
sb.Append(" " + _fileNames.GetDriverLoggingConfigCommand());
return(sb.ToString());
}
/// <summary>
/// Serializes the application parameters to reef/local/app-submission-params.json.
/// </summary>
internal void SerializeAppFile(AppParameters appParameters, IInjector paramInjector, string localDriverFolderPath)
{
var serializedArgs = SerializeAppArgsToBytes(appParameters, paramInjector, localDriverFolderPath);
var submissionAppArgsFilePath = Path.Combine(
localDriverFolderPath, _fileNames.GetLocalFolderPath(), _fileNames.GetAppSubmissionParametersFile());
using (var jobArgsFileStream = new FileStream(submissionAppArgsFilePath, FileMode.CreateNew))
{
jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
}
/// <summary>
/// Serializes the application parameters to reef/local/app-submission-params.json.
/// </summary>
internal string SerializeAppFile(AppParameters appParameters, IInjector paramInjector, string driverFolderPath)
{
var serializedArgs = SerializeAppArgsToBytes(appParameters, paramInjector);
var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetAppSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return(submissionArgsFilePath);
}