/// <summary>
/// Builds the command to be submitted to YARNRM
/// </summary>
/// <returns>Command string</returns>
public string GetJobSubmissionCommand()
{
var sb = new StringBuilder();
sb.Append(_fileNames.GetBridgeExePath());
sb.Append(" " + JavaExe);
sb.Append(" " + JvmOptionsPermSize);
sb.Append(" " + string.Format(JvmOptionsMaxPermSizeFormat, _driverMaxPermSizeMB));
sb.Append(" " +
string.Format(JvmOptionsMaxMemoryAllocationPoolSizeFormat, _driverMaxMemoryAllocationPoolSizeMB));
sb.Append(" " + ClassPathToken);
var yarnClasspathList = new List <string>(_yarnCommandLineEnvironment.GetYarnClasspathList())
{
string.Format("{0}/{1}/*", _fileNames.GetReefFolderName(), _fileNames.GetLocalFolderName()),
string.Format("{0}/{1}/*", _fileNames.GetReefFolderName(), _fileNames.GetGlobalFolderName())
};
sb.Append(" " + string.Join(";", yarnClasspathList));
sb.Append(" " + ProcReefProperty);
if (_enableDebugLogging)
{
sb.Append(" " + JavaLoggingProperty);
}
sb.Append(" " + LauncherClassName);
sb.Append(" " + _fileNames.GetJobSubmissionParametersFile());
sb.Append(" " +
string.Format("{0}/{1}/{2}",
_fileNames.GetReefFolderName(),
_fileNames.GetLocalFolderName(),
_fileNames.GetAppSubmissionParametersFile()));
sb.Append(" " + _fileNames.GetDriverLoggingConfigCommand());
return(sb.ToString());
}
/// <summary>
/// Creates the driver folder structure in this given folder as the root
/// </summary>
/// <param name="appParameters">Job submission information</param>
/// <param name="driverFolderPath">Driver folder path</param>
internal void CreateDefaultFolderStructure(AppParameters appParameters, string driverFolderPath)
{
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName()));
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath()));
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath()));
var resourceHelper = new ResourceHelper(typeof(DriverFolderPreparationHelper).Assembly);
foreach (var fileResources in ResourceHelper.FileResources)
{
var fileName = resourceHelper.GetString(fileResources.Key);
if (ResourceHelper.ClrDriverFullName == fileResources.Key)
{
fileName = Path.Combine(driverFolderPath, _fileNames.GetBridgeExePath());
}
if (!File.Exists(fileName))
{
File.WriteAllBytes(fileName, resourceHelper.GetBytes(fileResources.Value));
}
}
// generate .config file for bridge executable
var config = DefaultDriverConfigurationFileContents;
if (!string.IsNullOrEmpty(appParameters.DriverConfigurationFileContents))
{
config = appParameters.DriverConfigurationFileContents;
}
File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetBridgeExeConfigPath()), config);
// generate .config file for Evaluator executable
File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath(), EvaluatorExecutable),
DefaultDriverConfigurationFileContents);
}
/// <summary>
/// Creates the driver folder structure in this given folder as the root
/// </summary>
/// <param name="jobSubmission">Job submission information</param>
/// <param name="driverFolderPath">Driver folder path</param>
internal void CreateDefaultFolderStructure(IJobSubmission jobSubmission, string driverFolderPath)
{
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName()));
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath()));
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath()));
var resourceHelper = new ResourceHelper(typeof(DriverFolderPreparationHelper).Assembly);
foreach (var fileResources in clientFileResources)
{
var fileName = resourceHelper.GetString(fileResources.Item1);
if (ClrDriverFullName == fileResources.Item1)
{
fileName = Path.Combine(driverFolderPath, _fileNames.GetBridgeExePath());
}
File.WriteAllBytes(fileName, resourceHelper.GetBytes(fileResources.Item2));
}
var config = DefaultDriverConfigurationFileContents;
if (!string.IsNullOrEmpty(jobSubmission.DriverConfigurationFileContents))
{
config = jobSubmission.DriverConfigurationFileContents;
}
File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetBridgeExeConfigPath()), config);
}
private void SerializeJobFile(string localDriverFolderPath, AvroAzureBatchJobSubmissionParameters jobParameters)
{
var serializedArgs = AvroJsonSerializer <AvroAzureBatchJobSubmissionParameters> .ToBytes(jobParameters);
var submissionJobArgsFilePath = Path.Combine(localDriverFolderPath,
_fileNames.GetReefFolderName(), _fileNames.GetJobSubmissionParametersFile());
using (var jobArgsFileStream = new FileStream(submissionJobArgsFilePath, FileMode.CreateNew))
{
jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
}
public string BuildDriverCommand(int driverMemory)
{
var sb = new StringBuilder();
sb.Append(_fileNames.GetBridgeExePath())
.Append(" " + JavaExe)
.Append(" " + string.Format(JvmOptionsMaxMemoryAllocationPoolSizeFormat, driverMemory))
.Append(" " + JvmOptionsPermSize)
.Append(" " + JvmOptionsMaxPermSizeFormat)
.Append(" " + ClassPathToken)
.Append(" " + GetDriverClasspath())
.Append(" " + ProcReefProperty)
.Append(" " + LauncherClassName)
.Append(" " + Path.Combine(_fileNames.GetReefFolderName(), _fileNames.GetJobSubmissionParametersFile()));
return(string.Format(_osCommandFormat, _commandPrefix + sb.ToString()));
}
public string CreateArchiveToUpload(string folderPath)
{
string archivePath = Path.Combine(folderPath, Path.GetDirectoryName(folderPath) + ".zip");
string reefFolder = Path.Combine(folderPath, _reefFileNames.GetReefFolderName());
if (!Directory.Exists(reefFolder))
{
throw new DirectoryNotFoundException("Cannot find directory " + reefFolder);
}
if (File.Exists(archivePath))
{
throw new InvalidOperationException("Archive file already exists " + archivePath);
}
ZipFile.CreateFromDirectory(reefFolder, archivePath);
return(archivePath);
}
/// <summary>
/// Creates the driver folder structure in this given folder as the root
/// </summary>
/// <param name="driverFolderPath"></param>
internal void CreateDefaultFolderStructure(string driverFolderPath)
{
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName()));
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath()));
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath()));
}
public async Task <JobResource> UploadArchiveResourceAsync(string driverLocalFolderPath, string remoteUploadDirectoryPath)
{
driverLocalFolderPath = driverLocalFolderPath.TrimEnd('\\') + @"\";
var driverUploadPath = remoteUploadDirectoryPath.TrimEnd('/') + @"/";
Log.Log(Level.Info, "DriverFolderPath: {0} DriverUploadPath: {1}", driverLocalFolderPath, driverUploadPath);
var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath);
return(await UploadResourceAndGetInfoAsync(archivePath, ResourceType.ARCHIVE, driverUploadPath, _reefFileNames.GetReefFolderName()));
}