/// <summary>
/// Serializes the application parameters to reef/local/app-submission-params.json.
/// </summary>
internal void SerializeAppFile(AppParameters appParameters, IInjector paramInjector, string localDriverFolderPath)
{
var serializedArgs = SerializeAppArgsToBytes(appParameters, paramInjector, localDriverFolderPath);
var submissionAppArgsFilePath = Path.Combine(
localDriverFolderPath, _fileNames.GetLocalFolderPath(), _fileNames.GetAppSubmissionParametersFile());
using (var jobArgsFileStream = new FileStream(submissionAppArgsFilePath, FileMode.CreateNew))
{
jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
}
/// <summary>
/// Serializes the application parameters to reef/local/app-submission-params.json.
/// </summary>
internal string SerializeAppFile(AppParameters appParameters, IInjector paramInjector, string driverFolderPath)
{
var serializedArgs = SerializeAppArgsToBytes(appParameters, paramInjector);
var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetAppSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return submissionArgsFilePath;
}
internal byte[] SerializeAppArgsToBytes(AppParameters appParameters, IInjector paramInjector)
{
var avroAppSubmissionParameters = new AvroAppSubmissionParameters
{
tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>()
};
var avroYarnAppSubmissionParameters = new AvroYarnAppSubmissionParameters
{
sharedAppSubmissionParameters = avroAppSubmissionParameters,
driverRecoveryTimeout = paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>()
};
return AvroJsonSerializer<AvroYarnAppSubmissionParameters>.ToBytes(avroYarnAppSubmissionParameters);
}
/// <summary>
/// Prepares the working directory for a Driver in driverFolderPath.
/// </summary>
/// <param name="appParameters"></param>
/// <param name="driverFolderPath"></param>
internal void PrepareDriverFolder(AppParameters appParameters, string driverFolderPath)
{
Logger.Log(Level.Verbose, "Preparing Driver filesystem layout in " + driverFolderPath);
// Setup the folder structure
CreateDefaultFolderStructure(appParameters, driverFolderPath);
// Add the appParameters into that folder structure
_fileSets.AddJobFiles(appParameters);
// Create the driver configuration
CreateDriverConfiguration(appParameters, driverFolderPath);
// Add the REEF assemblies
AddAssemblies();
// Initiate the final copy
_fileSets.CopyToDriverFolder(driverFolderPath);
Logger.Log(Level.Info, "Done preparing Driver filesystem layout in " + driverFolderPath);
}
internal static JobRequest FromJobSubmission(IJobSubmission jobSubmission)
{
return(new JobRequest(
JobParameters.FromJobSubmission(jobSubmission), AppParameters.FromJobSubmission(jobSubmission)));
}
internal JobRequest(JobParameters jobParameters, AppParameters appParameters)
{
_jobParameters = jobParameters;
_appParameters = appParameters;
}
internal JobRequest(JobParameters jobParameters, AppParameters appParameters)
{
_jobParameters = jobParameters;
_appParameters = appParameters;
}
/// <summary>
/// Creates the driver folder structure in this given folder as the root
/// </summary>
/// <param name="appParameters">Job submission information</param>
/// <param name="driverFolderPath">Driver folder path</param>
internal void CreateDefaultFolderStructure(AppParameters appParameters, string driverFolderPath)
{
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName()));
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath()));
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath()));
var resourceHelper = new ResourceHelper(typeof(DriverFolderPreparationHelper).Assembly);
foreach (var fileResources in ResourceHelper.FileResources)
{
var fileName = resourceHelper.GetString(fileResources.Key);
if (ResourceHelper.ClrDriverFullName == fileResources.Key)
{
fileName = Path.Combine(driverFolderPath, _fileNames.GetBridgeExePath());
}
if (!File.Exists(fileName))
{
File.WriteAllBytes(fileName, resourceHelper.GetBytes(fileResources.Value));
}
}
// generate .config file for bridge executable
var config = DefaultDriverConfigurationFileContents;
if (!string.IsNullOrEmpty(appParameters.DriverConfigurationFileContents))
{
config = appParameters.DriverConfigurationFileContents;
}
File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetBridgeExeConfigPath()), config);
// generate .config file for Evaluator executable
File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath(), EvaluatorExecutable),
DefaultDriverConfigurationFileContents);
}
/// <summary>
/// Merges the Configurations in appParameters and serializes them into the right place within driverFolderPath,
/// assuming
/// that points to a Driver's working directory.
/// </summary>
/// <param name="appParameters"></param>
/// <param name="driverFolderPath"></param>
internal void CreateDriverConfiguration(AppParameters appParameters, string driverFolderPath)
{
var driverConfigurations = _driverConfigurationProviders.Select(configurationProvider => configurationProvider.GetConfiguration()).ToList();
var driverConfiguration = Configurations.Merge(driverConfigurations.Concat(appParameters.DriverConfigurations).ToArray());
_configurationSerializer.ToFile(driverConfiguration,
Path.Combine(driverFolderPath, _fileNames.GetClrDriverConfigurationPath()));
}
private string CreateBootstrapAvroAppConfig(AppParameters appParameters, string driverFolder)
{
var paramInjector = TangFactory.GetTang().NewInjector(appParameters.DriverConfigurations.ToArray());
var bootstrapAppArgs = new AvroAppSubmissionParameters
{
tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>(),
};
var avroLocalBootstrapAppArgs = new AvroLocalAppSubmissionParameters
{
sharedAppSubmissionParameters = bootstrapAppArgs,
maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators
};
var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetAppSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer<AvroLocalAppSubmissionParameters>.ToBytes(avroLocalBootstrapAppArgs);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return submissionArgsFilePath;
}