/// <summary>
/// Adds all the files referenced in the given JobSubmission
/// </summary>
/// <param name="submission"></param>
internal void AddJobFiles(IJobSubmission submission)
{
AddToGlobalFiles(submission.GlobalFiles);
AddToGlobalFiles(submission.GlobalAssemblies);
AddToLocalFiles(submission.LocalFiles);
AddToLocalFiles(submission.LocalAssemblies);
}
private string CreateBootstrapAvroConfig(IJobSubmission jobSubmission, string driverFolder)
{
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
var bootstrapArgs = new AvroJobSubmissionParameters
{
jobSubmissionFolder = driverFolder,
jobId = jobSubmission.JobIdentifier,
tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(),
};
var avroLocalBootstrapArgs = new AvroLocalJobSubmissionParameters
{
sharedJobSubmissionParameters = bootstrapArgs,
maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators
};
var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroLocalJobSubmissionParameters> .ToBytes(avroLocalBootstrapArgs);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return(submissionArgsFilePath);
}
/// <summary>
/// Creates the driver folder structure in this given folder as the root
/// </summary>
/// <param name="jobSubmission">Job submission information</param>
/// <param name="driverFolderPath">Driver folder path</param>
internal void CreateDefaultFolderStructure(IJobSubmission jobSubmission, string driverFolderPath)
{
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName()));
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath()));
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath()));
var resourceHelper = new ResourceHelper(typeof(DriverFolderPreparationHelper).Assembly);
foreach (var fileResources in clientFileResources)
{
var fileName = resourceHelper.GetString(fileResources.Item1);
if (ClrDriverFullName == fileResources.Item1)
{
fileName = Path.Combine(driverFolderPath, _fileNames.GetBridgeExePath());
}
File.WriteAllBytes(fileName, resourceHelper.GetBytes(fileResources.Item2));
}
var config = DefaultDriverConfigurationFileContents;
if (!string.IsNullOrEmpty(jobSubmission.DriverConfigurationFileContents))
{
config = jobSubmission.DriverConfigurationFileContents;
}
File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetBridgeExeConfigPath()), config);
}
public void Submit(IJobSubmission jobSubmission)
{
var driverFolder = PrepareDriverFolder(jobSubmission);
var submissionArgsFilePath = CreateBootstrapAvroConfig(jobSubmission, driverFolder);
_javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
Logger.Log(Level.Info, "Submitted the Driver for execution.");
}
public void Submit(IJobSubmission jobSubmission)
{
// Prepare the job submission folder
var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier);
Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath);
Launch(jobSubmission, driverFolderPath);
}
public void Submit(IJobSubmission jobSubmission)
{
// Prepare the job submission folder
var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier);
Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath);
Launch(jobSubmission, driverFolderPath);
}
internal static JobParameters FromJobSubmission(IJobSubmission jobSubmission)
{
return new JobParameters(
jobSubmission.JobIdentifier,
jobSubmission.MaxApplicationSubmissions,
jobSubmission.DriverMemory,
null,
null);
}
/// <summary>
/// Merges the Configurations in jobSubmission and serializes them into the right place within driverFolderPath,
/// assuming
/// that points to a Driver's working directory.
/// </summary>
/// <param name="jobSubmission"></param>
/// <param name="driverFolderPath"></param>
internal void CreateDriverConfiguration(IJobSubmission jobSubmission, string driverFolderPath)
{
var driverConfiguration = Configurations.Merge(jobSubmission.DriverConfigurations.ToArray());
_configurationSerializer.ToFile(driverConfiguration,
Path.Combine(driverFolderPath, _fileNames.GetClrDriverConfigurationPath()));
// TODO: Remove once we cleaned up the Evaluator to not expect this [REEF-216]
_configurationSerializer.ToFile(driverConfiguration,
Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath(), Constants.ClrBridgeRuntimeConfiguration));
}
/// <summary>
/// Serializes the application parameters to reef/local/app-submission-params.json.
/// </summary>
internal void SerializeAppFile(IJobSubmission jobSubmission, IInjector paramInjector, string localDriverFolderPath)
{
var serializedArgs = SerializeAppArgsToBytes(jobSubmission, paramInjector, localDriverFolderPath);
var submissionAppArgsFilePath = Path.Combine(
localDriverFolderPath, _fileNames.GetLocalFolderPath(), _fileNames.GetAppSubmissionParametersFile());
using (var jobArgsFileStream = new FileStream(submissionAppArgsFilePath, FileMode.CreateNew))
{
jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
}
private string PrepareDriverFolder(IJobSubmission jobSubmission)
{
// Prepare the job submission folder
var jobFolder = CreateJobFolder(jobSubmission.JobIdentifier);
var driverFolder = Path.Combine(jobFolder, DriverFolderName);
Logger.Log(Level.Info, "Preparing driver folder in " + driverFolder);
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolder);
return(driverFolder);
}
/// <summary>
/// Serializes the application parameters to reef/local/app-submission-params.json.
/// </summary>
internal string SerializeAppFile(IJobSubmission jobSubmission, IInjector paramInjector, string driverFolderPath)
{
var serializedArgs = SerializeAppArgsToBytes(jobSubmission, paramInjector);
var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetAppSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return submissionArgsFilePath;
}
/// <summary>
/// Serializes the job parameters to job-submission-params.json.
/// </summary>
internal void SerializeJobFile(IJobSubmission jobSubmission, string localDriverFolderPath, string jobSubmissionDirectory)
{
var serializedArgs = SerializeJobArgsToBytes(jobSubmission, localDriverFolderPath, jobSubmissionDirectory);
var submissionJobArgsFilePath = Path.Combine(localDriverFolderPath,
_fileNames.GetJobSubmissionParametersFile());
using (var jobArgsFileStream = new FileStream(submissionJobArgsFilePath, FileMode.CreateNew))
{
jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
}
public IDriverHttpEndpoint SubmitAndGetDriverUrl(IJobSubmission jobSubmission)
{
// Prepare the job submission folder
var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier);
Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath);
Launch(jobSubmission, driverFolderPath);
var pointerFileName = Path.Combine(driverFolderPath, _fileNames.DriverHttpEndpoint);
var httpClient = new HttpClientHelper();
_driverUrl = httpClient.GetDriverUrlForYarn(pointerFileName);
return httpClient;
}
public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
{
var driverFolder = PrepareDriverFolder(jobSubmission);
var submissionArgsFilePath = CreateBootstrapAvroConfig(jobSubmission, driverFolder);
Task.Run(() => _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath));
var fileName = Path.Combine(driverFolder, _fileNames.DriverHttpEndpoint);
JobSubmissionResult result = new LocalJobSubmissionResult(this, fileName);
var msg = string.Format(CultureInfo.CurrentCulture,
"Submitted the Driver for execution. Returned driverUrl is: {0}.", result.DriverUrl);
Logger.Log(Level.Info, msg);
return(result);
}
private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
{
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);
// TODO: Remove this when we have a generalized way to pass config to java
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(),
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
driverMemory = jobSubmission.DriverMemory,
driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
maxApplicationSubmissions = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(),
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters
};
var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
// Submit the driver
_javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
}
public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
{
// Prepare the job submission folder
var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier);
Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath);
Launch(jobSubmission, driverFolderPath);
var pointerFileName = Path.Combine(driverFolderPath, _fileNames.DriverHttpEndpoint);
var jobSubmitionResultImpl = new YarnJobSubmissionResult(this, pointerFileName);
var msg = string.Format(CultureInfo.CurrentCulture,
"Submitted the Driver for execution. Returned driverUrl is: {0}, appId is {1}.",
jobSubmitionResultImpl.DriverUrl, jobSubmitionResultImpl.AppId);
Logger.Log(Level.Info, msg);
return jobSubmitionResultImpl;
}
internal byte[] SerializeAppArgsToBytes(IJobSubmission jobSubmission, IInjector paramInjector, string localDriverFolderPath)
{
var avroAppSubmissionParameters = new AvroAppSubmissionParameters
{
tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>()
};
var avroYarnAppSubmissionParameters = new AvroYarnAppSubmissionParameters
{
sharedAppSubmissionParameters = avroAppSubmissionParameters,
driverMemory = jobSubmission.DriverMemory,
driverRecoveryTimeout =
paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
};
return AvroJsonSerializer<AvroYarnAppSubmissionParameters>.ToBytes(avroYarnAppSubmissionParameters);
}
public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
{
// Prepare the job submission folder
var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier);
Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath);
Launch(jobSubmission, driverFolderPath);
var pointerFileName = Path.Combine(driverFolderPath, _fileNames.DriverHttpEndpoint);
var jobSubmitionResultImpl = new YarnJobSubmissionResult(this, pointerFileName);
var msg = string.Format(CultureInfo.CurrentCulture,
"Submitted the Driver for execution. Returned driverUrl is: {0}, appId is {1}.",
jobSubmitionResultImpl.DriverUrl, jobSubmitionResultImpl.AppId);
Logger.Log(Level.Info, msg);
return(jobSubmitionResultImpl);
}
private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
{
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);
//TODO: Remove this when we have a generalized way to pass config to java
var javaParams = TangFactory.GetTang()
.NewInjector(jobSubmission.DriverConfigurations.ToArray())
.GetInstance<ClrClient2JavaClientCuratedParameters>();
// Submit the driver
_javaClientLauncher.Launch(JavaClassName, driverFolderPath, jobSubmission.JobIdentifier,
jobSubmission.DriverMemory.ToString(),
javaParams.TcpPortRangeStart.ToString(),
javaParams.TcpPortRangeCount.ToString(),
javaParams.TcpPortRangeTryCount.ToString(),
javaParams.MaxApplicationSubmissions.ToString(),
javaParams.DriverRestartEvaluatorRecoverySeconds.ToString()
);
Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
}
/// <summary>
/// Prepares the working directory for a Driver in driverFolderPath.
/// </summary>
/// <param name="jobSubmission"></param>
/// <param name="driverFolderPath"></param>
internal void PrepareDriverFolder(IJobSubmission jobSubmission, string driverFolderPath)
{
Logger.Log(Level.Info, "Preparing Driver filesystem layout in " + driverFolderPath);
// Setup the folder structure
CreateDefaultFolderStructure(jobSubmission, driverFolderPath);
// Add the jobSubmission into that folder structure
_fileSets.AddJobFiles(jobSubmission);
// Create the driver configuration
CreateDriverConfiguration(jobSubmission, driverFolderPath);
// Add the REEF assemblies
AddAssemblies();
// Initiate the final copy
_fileSets.CopyToDriverFolder(driverFolderPath);
Logger.Log(Level.Info, "Done preparing Driver filesystem layout in " + driverFolderPath);
}
/// <summary>
/// Prepares the working directory for a Driver in driverFolderPath.
/// </summary>
/// <param name="jobSubmission"></param>
/// <param name="driverFolderPath"></param>
internal void PrepareDriverFolder(IJobSubmission jobSubmission, string driverFolderPath)
{
Logger.Log(Level.Info, "Preparing Driver filesystem layout in " + driverFolderPath);
// Setup the folder structure
CreateDefaultFolderStructure(driverFolderPath);
// Add the jobSubmission into that folder structure
_fileSets.AddJobFiles(jobSubmission);
// Create the driver configuration
CreateDriverConfiguration(jobSubmission, driverFolderPath);
// Add the REEF assemblies
AddAssemblies();
// Initiate the final copy
_fileSets.CopyToDriverFolder(driverFolderPath);
Logger.Log(Level.Info, "Done preparing Driver filesystem layout in " + driverFolderPath);
}
public void Submit(IJobSubmission jobSubmission)
{
// Prepare the job submission folder
var driverFolderPath = CreateDriverFolder(jobSubmission.JobIdentifier);
Logger.Log(Level.Info, "Preparing driver folder in " + driverFolderPath);
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);
//TODO: Remove this when we have a generalized way to pass config to java
var javaParams = TangFactory.GetTang()
.NewInjector(jobSubmission.DriverConfigurations.ToArray())
.GetInstance<ClrClient2JavaClientCuratedParameters>();
// Submit the driver
_javaClientLauncher.Launch(JavaClassName, driverFolderPath, jobSubmission.JobIdentifier,
jobSubmission.DriverMemory.ToString(),
javaParams.TcpPortRangeStart.ToString(),
javaParams.TcpPortRangeCount.ToString(),
javaParams.TcpPortRangeTryCount.ToString()
);
Logger.Log(Level.Info, "Submitted the Driver for execution.");
}
public void Submit(IJobSubmission jobSubmission)
{
// Prepare the job submission folder
var jobFolder = CreateJobFolder(jobSubmission.JobIdentifier);
var driverFolder = Path.Combine(jobFolder, DriverFolderName);
Logger.Log(Level.Info, "Preparing driver folder in " + driverFolder);
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolder);
//TODO: Remove this when we have a generalized way to pass config to java
var javaParams = TangFactory.GetTang()
.NewInjector(jobSubmission.DriverConfigurations.ToArray())
.GetInstance <ClrClient2JavaClientCuratedParameters>();
_javaClientLauncher.Launch(JavaClassName, driverFolder, jobSubmission.JobIdentifier,
_numberOfEvaluators.ToString(),
javaParams.TcpPortRangeStart.ToString(),
javaParams.TcpPortRangeCount.ToString(),
javaParams.TcpPortRangeTryCount.ToString()
);
Logger.Log(Level.Info, "Submitted the Driver for execution.");
}
public void Submit(IJobSubmission jobSubmission)
{
var driverFolder = PrepareDriverFolder(jobSubmission);
var submissionArgsFilePath = CreateBootstrapAvroConfig(jobSubmission, driverFolder);
_javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
Logger.Log(Level.Info, "Submitted the Driver for execution.");
}
private string PrepareDriverFolder(IJobSubmission jobSubmission)
{
// Prepare the job submission folder
var jobFolder = CreateJobFolder(jobSubmission.JobIdentifier);
var driverFolder = Path.Combine(jobFolder, DriverFolderName);
Logger.Log(Level.Info, "Preparing driver folder in " + driverFolder);
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolder);
return driverFolder;
}
private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
{
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);
// TODO: Remove this when we have a generalized way to pass config to java
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
var submissionJobArgsFilePath = _paramSerializer.SerializeJobFile(jobSubmission, driverFolderPath);
var submissionAppArgsFilePath = _paramSerializer.SerializeAppFile(jobSubmission, paramInjector, driverFolderPath);
// Submit the driver
_javaClientLauncher.Launch(JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath);
Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
}
internal static JobRequest FromJobSubmission(IJobSubmission jobSubmission)
{
return new JobRequest(
JobParameters.FromJobSubmission(jobSubmission), AppParameters.FromJobSubmission(jobSubmission));
}
private SubmitApplication CreateApplicationSubmissionRequest(
IJobSubmission jobSubmission,
string appId,
int maxApplicationSubmissions,
JobResource jobResource)
{
string command = _yarnJobCommandProvider.GetJobSubmissionCommand();
Log.Log(Level.Verbose, "Command for YARN: {0}", command);
Log.Log(Level.Verbose, "ApplicationID: {0}", appId);
Log.Log(Level.Verbose, "MaxApplicationSubmissions: {0}", maxApplicationSubmissions);
Log.Log(Level.Verbose, "Driver archive location: {0}", jobResource.RemoteUploadPath);
var submitApplication = new SubmitApplication
{
ApplicationId = appId,
ApplicationName = jobSubmission.JobIdentifier,
AmResource = new Resouce
{
MemoryMB = jobSubmission.DriverMemory,
VCores = 1 // keeping parity with existing code
},
MaxAppAttempts = maxApplicationSubmissions,
ApplicationType = REEFApplicationType,
KeepContainersAcrossApplicationAttempts = true,
Queue = @"default", // keeping parity with existing code
Priority = 1, // keeping parity with existing code
UnmanagedAM = false,
AmContainerSpec = new AmContainerSpec
{
LocalResources = new LocalResources
{
Entries = new[]
{
new KeyValuePair<string, LocalResourcesValue>
{
Key = _fileNames.GetReefFolderName(),
Value = new LocalResourcesValue
{
Resource = jobResource.RemoteUploadPath,
Type = ResourceType.ARCHIVE,
Visibility = Visibility.APPLICATION,
Size = jobResource.ResourceSize,
Timestamp = jobResource.LastModificationUnixTimestamp
}
}
}
},
Commands = new Commands
{
Command = command
}
}
};
return submitApplication;
}
public void Submit(IJobSubmission jobSubmission)
{
string jobId = jobSubmission.JobIdentifier;
// todo: Future client interface should be async.
// Using GetAwaiter().GetResult() instead of .Result to avoid exception
// getting wrapped in AggregateException.
var newApplication = _yarnRMClient.CreateNewApplicationAsync().GetAwaiter().GetResult();
string applicationId = newApplication.ApplicationId;
// create job submission remote path
string jobSubmissionDirectory =
_jobSubmissionDirectoryProvider.GetJobSubmissionRemoteDirectory(applicationId);
// create local driver folder.
var localDriverFolderPath = CreateDriverFolder(jobId, applicationId);
try
{
Log.Log(Level.Verbose, "Preparing driver folder in {0}", localDriverFolderPath);
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, localDriverFolderPath);
// prepare configuration
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
int maxApplicationSubmissions =
paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>();
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>(),
jobSubmissionFolder = localDriverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
driverMemory = jobSubmission.DriverMemory,
driverRecoveryTimeout =
paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
jobSubmissionDirectoryPrefix = jobSubmissionDirectory,
dfsJobSubmissionFolder = jobSubmissionDirectory,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var submissionArgsFilePath = Path.Combine(localDriverFolderPath,
_fileNames.GetLocalFolderPath(),
_fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs =
AvroJsonSerializer<AvroYarnJobSubmissionParameters>.ToBytes(avroYarnJobSubmissionParameters);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
// upload prepared folder to DFS
var jobResource = _jobResourceUploader.UploadJobResource(localDriverFolderPath, jobSubmissionDirectory);
// submit job
Log.Log(Level.Verbose, @"Assigned application id {0}", applicationId);
var submissionReq = CreateApplicationSubmissionRequest(jobSubmission,
applicationId,
maxApplicationSubmissions,
jobResource);
var submittedApplication = _yarnRMClient.SubmitApplicationAsync(submissionReq).GetAwaiter().GetResult();
Log.Log(Level.Info, @"Submitted application {0}", submittedApplication.Id);
}
finally
{
if (Directory.Exists(localDriverFolderPath))
{
Directory.Delete(localDriverFolderPath, recursive: true);
}
}
}
public void Submit(IJobSubmission jobSubmission)
{
Submit(JobRequest.FromJobSubmission(jobSubmission));
}
public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
{
var driverFolder = PrepareDriverFolder(jobSubmission);
var submissionArgsFilePath = CreateBootstrapAvroConfig(jobSubmission, driverFolder);
Task.Run(() => _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath));
var fileName = Path.Combine(driverFolder, _fileNames.DriverHttpEndpoint);
JobSubmissionResult result = new LocalJobSubmissionResult(this, fileName);
var msg = string.Format(CultureInfo.CurrentCulture,
"Submitted the Driver for execution. Returned driverUrl is: {0}.", result.DriverUrl);
Logger.Log(Level.Info, msg);
return result;
}
/// <summary>
/// Creates the driver folder structure in this given folder as the root
/// </summary>
/// <param name="jobSubmission">Job submission information</param>
/// <param name="driverFolderPath">Driver folder path</param>
internal void CreateDefaultFolderStructure(IJobSubmission jobSubmission, string driverFolderPath)
{
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetReefFolderName()));
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetLocalFolderPath()));
Directory.CreateDirectory(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath()));
var resourceHelper = new ResourceHelper(typeof(DriverFolderPreparationHelper).Assembly);
foreach (var fileResources in clientFileResources)
{
var fileName = resourceHelper.GetString(fileResources.Item1);
if (ClrDriverFullName == fileResources.Item1)
{
fileName = Path.Combine(driverFolderPath, _fileNames.GetBridgeExePath());
}
File.WriteAllBytes(fileName, resourceHelper.GetBytes(fileResources.Item2));
}
var config = DefaultDriverConfigurationFileContents;
if (!string.IsNullOrEmpty(jobSubmission.DriverConfigurationFileContents))
{
config = jobSubmission.DriverConfigurationFileContents;
}
File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetBridgeExeConfigPath()), config);
}
internal static AppParameters FromJobSubmission(IJobSubmission jobSubmission)
{
return(new AppParameters(jobSubmission.DriverConfigurations, jobSubmission.GlobalAssemblies, jobSubmission.GlobalFiles,
jobSubmission.LocalAssemblies, jobSubmission.LocalFiles, jobSubmission.DriverConfigurationFileContents));
}
public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
{
return(SubmitAndGetJobStatus(JobRequest.FromJobSubmission(jobSubmission)));
}
internal static JobRequest FromJobSubmission(IJobSubmission jobSubmission)
{
return(new JobRequest(
JobParameters.FromJobSubmission(jobSubmission), AppParameters.FromJobSubmission(jobSubmission)));
}
private string CreateBootstrapAvroConfig(IJobSubmission jobSubmission, string driverFolder)
{
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
var bootstrapArgs = new AvroJobSubmissionParameters
{
jobSubmissionFolder = driverFolder,
jobId = jobSubmission.JobIdentifier,
tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>(),
};
var avroLocalBootstrapArgs = new AvroLocalJobSubmissionParameters
{
sharedJobSubmissionParameters = bootstrapArgs,
maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators
};
var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer<AvroLocalJobSubmissionParameters>.ToBytes(avroLocalBootstrapArgs);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return submissionArgsFilePath;
}
private string CreateBootstrapAvroJobConfig(IJobSubmission jobSubmission, string driverFolder)
{
var bootstrapJobArgs = new AvroJobSubmissionParameters
{
jobSubmissionFolder = driverFolder,
jobId = jobSubmission.JobIdentifier,
};
var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer<AvroJobSubmissionParameters>.ToBytes(bootstrapJobArgs);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return submissionArgsFilePath;
}
internal byte[] SerializeJobArgsToBytes(IJobSubmission jobSubmission, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters
};
return AvroJsonSerializer<AvroYarnClusterJobSubmissionParameters>.ToBytes(avroYarnClusterJobSubmissionParameters);
}
public void Submit(IJobSubmission jobSubmission)
{
Submit(JobRequest.FromJobSubmission(jobSubmission));
}
private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
{
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);
// TODO: Remove this when we have a generalized way to pass config to java
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
tcpBeginPort = paramInjector.GetNamedInstance<TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance<TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance<TcpPortRangeTryCount, int>(),
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
driverMemory = jobSubmission.DriverMemory,
driverRecoveryTimeout = paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
maxApplicationSubmissions = paramInjector.GetNamedInstance<DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(),
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters
};
var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer<AvroYarnClusterJobSubmissionParameters>.ToBytes(avroYarnClusterJobSubmissionParameters);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
// Submit the driver
_javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
}
public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
{
return SubmitAndGetJobStatus(JobRequest.FromJobSubmission(jobSubmission));
}
internal static AppParameters FromJobSubmission(IJobSubmission jobSubmission)
{
return new AppParameters(jobSubmission.DriverConfigurations, jobSubmission.GlobalAssemblies, jobSubmission.GlobalFiles,
jobSubmission.LocalAssemblies, jobSubmission.LocalFiles, jobSubmission.DriverConfigurationFileContents);
}
public IJobSubmissionResult SubmitAndGetJobStatus(IJobSubmission jobSubmission)
{
throw new NotSupportedException();
}
internal byte[] SerializeJobArgsToBytes(IJobSubmission jobSubmission, string localDriverFolderPath, string jobSubmissionDirectory)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
jobSubmissionFolder = localDriverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = jobSubmissionDirectory,
dfsJobSubmissionFolder = jobSubmissionDirectory,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
return AvroJsonSerializer<AvroYarnJobSubmissionParameters>.ToBytes(avroYarnJobSubmissionParameters);
}
/// <summary>
/// Merges the Configurations in jobSubmission and serializes them into the right place within driverFolderPath,
/// assuming
/// that points to a Driver's working directory.
/// </summary>
/// <param name="jobSubmission"></param>
/// <param name="driverFolderPath"></param>
internal void CreateDriverConfiguration(IJobSubmission jobSubmission, string driverFolderPath)
{
var driverConfiguration = Configurations.Merge(jobSubmission.DriverConfigurations.ToArray());
_configurationSerializer.ToFile(driverConfiguration,
Path.Combine(driverFolderPath, _fileNames.GetClrDriverConfigurationPath()));
// TODO: Remove once we cleaned up the Evaluator to not expect this [REEF-217]
_configurationSerializer.ToFile(driverConfiguration,
Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath(), _fileNames.GetClrBridgeConfigurationName()));
}
public IDriverHttpEndpoint SubmitAndGetDriverUrl(IJobSubmission jobSubmission)
{
// Prepare the job submission folder
var jobFolder = CreateJobFolder(jobSubmission.JobIdentifier);
var driverFolder = Path.Combine(jobFolder, DriverFolderName);
Logger.Log(Level.Info, "Preparing driver folder in " + driverFolder);
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolder);
//TODO: Remove this when we have a generalized way to pass config to java
var javaParams = TangFactory.GetTang()
.NewInjector(jobSubmission.DriverConfigurations.ToArray())
.GetInstance<ClrClient2JavaClientCuratedParameters>();
Task.Run(() =>
_javaClientLauncher.Launch(JavaClassName, driverFolder, jobSubmission.JobIdentifier,
_numberOfEvaluators.ToString(),
javaParams.TcpPortRangeStart.ToString(),
javaParams.TcpPortRangeCount.ToString(),
javaParams.TcpPortRangeTryCount.ToString()
));
var fileName = Path.Combine(driverFolder, _fileNames.DriverHttpEndpoint);
HttpClientHelper helper = new HttpClientHelper();
_driverUrl = helper.GetDriverUrlForLocalRuntime(fileName);
Logger.Log(Level.Info, "Submitted the Driver for execution. Returned driverUrl is: " + _driverUrl);
return helper;
}
internal static JobParameters FromJobSubmission(IJobSubmission jobSubmission)
{
return(new JobParameters(
jobSubmission.JobIdentifier, jobSubmission.MaxApplicationSubmissions, jobSubmission.DriverMemory));
}
