private string CreateBootstrapAvroAppConfig(AppParameters appParameters, string driverFolder) { var paramInjector = TangFactory.GetTang().NewInjector(appParameters.DriverConfigurations.ToArray()); var bootstrapAppArgs = new AvroAppSubmissionParameters { tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(), tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(), tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(), }; var avroLocalBootstrapAppArgs = new AvroLocalAppSubmissionParameters { sharedAppSubmissionParameters = bootstrapAppArgs, maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators }; var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetAppSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { var serializedArgs = AvroJsonSerializer <AvroLocalAppSubmissionParameters> .ToBytes(avroLocalBootstrapAppArgs); argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } return(submissionArgsFilePath); }
/// <summary> /// Builds the command to be submitted to YARNRM /// </summary> /// <returns>Command string</returns> public string GetJobSubmissionCommand() { var sb = new StringBuilder(); sb.Append(_fileNames.GetBridgeExePath()); sb.Append(" " + JavaExe); sb.Append(" " + JvmOptionsPermSize); sb.Append(" " + string.Format(JvmOptionsMaxPermSizeFormat, _driverMaxPermSizeMB)); sb.Append(" " + string.Format(JvmOptionsMaxMemoryAllocationPoolSizeFormat, _driverMaxMemoryAllocationPoolSizeMB)); sb.Append(" " + ClassPathToken); var yarnClasspathList = new List <string>(_yarnCommandLineEnvironment.GetYarnClasspathList()) { string.Format("{0}/{1}/*", _fileNames.GetReefFolderName(), _fileNames.GetLocalFolderName()), string.Format("{0}/{1}/*", _fileNames.GetReefFolderName(), _fileNames.GetGlobalFolderName()) }; sb.Append(" " + string.Join(";", yarnClasspathList)); sb.Append(" " + ProcReefProperty); if (_enableDebugLogging) { sb.Append(" " + JavaLoggingProperty); } sb.Append(" " + LauncherClassName); sb.Append(" " + _fileNames.GetJobSubmissionParametersFile()); sb.Append(" " + string.Format("{0}/{1}/{2}", _fileNames.GetReefFolderName(), _fileNames.GetLocalFolderName(), _fileNames.GetAppSubmissionParametersFile())); sb.Append(" " + _fileNames.GetDriverLoggingConfigCommand()); return(sb.ToString()); }
/// <summary> /// Serializes the application parameters to reef/local/app-submission-params.json. /// </summary> internal void SerializeAppFile(AppParameters appParameters, IInjector paramInjector, string localDriverFolderPath) { var serializedArgs = SerializeAppArgsToBytes(appParameters, paramInjector, localDriverFolderPath); var submissionAppArgsFilePath = Path.Combine( localDriverFolderPath, _fileNames.GetLocalFolderPath(), _fileNames.GetAppSubmissionParametersFile()); using (var jobArgsFileStream = new FileStream(submissionAppArgsFilePath, FileMode.CreateNew)) { jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } }
/// <summary> /// Serializes the application parameters to reef/local/app-submission-params.json. /// </summary> internal string SerializeAppFile(AppParameters appParameters, IInjector paramInjector, string driverFolderPath) { var serializedArgs = SerializeAppArgsToBytes(appParameters, paramInjector); var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetAppSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } return(submissionArgsFilePath); }