internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { sharedJobSubmissionParameters = avroJobSubmissionParameters, fileSystemUrl = _fileSystemUrl, jobSubmissionDirectoryPrefix = _jobSubmissionPrefix }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { // TODO[JIRA REEF-1887] Deprecated. Remove in REEF 0.18. securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, driverMemory = jobParameters.DriverMemoryInMB, environmentVariablesMap = jobParameters.JobSubmissionEnvMap, maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions, driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ? _fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value, driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ? _fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value }; return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters)); }
private string CreateBootstrapAvroJobConfig(JobParameters jobParameters, string driverFolder) { var bootstrapJobArgs = new AvroJobSubmissionParameters { jobSubmissionFolder = driverFolder, jobId = jobParameters.JobIdentifier, }; var bootstrapLocalJobArgs = new AvroLocalJobSubmissionParameters { sharedJobSubmissionParameters = bootstrapJobArgs, driverStdoutFilePath = jobParameters.StdoutFilePath.IsPresent() ? jobParameters.StdoutFilePath.Value : _fileNames.GetDriverStdoutFileName(), driverStderrFilePath = jobParameters.StderrFilePath.IsPresent() ? jobParameters.StderrFilePath.Value : _fileNames.GetDriverStderrFileName() }; var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { var serializedArgs = AvroJsonSerializer <AvroLocalJobSubmissionParameters> .ToBytes(bootstrapLocalJobArgs); argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } return(submissionArgsFilePath); }
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { jobSubmissionDirectoryPrefix = _jobSubmissionPrefix, sharedJobSubmissionParameters = avroJobSubmissionParameters }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, driverMemory = jobParameters.DriverMemoryInMB, maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions, driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ? _fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value, driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ? _fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value }; return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters)); }
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { jobSubmissionDirectoryPrefix = _jobSubmissionPrefix, sharedJobSubmissionParameters = avroJobSubmissionParameters }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, driverMemory = jobParameters.DriverMemoryInMB, maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions }; return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters)); }
private string CreateBootstrapAvroConfig(IJobSubmission jobSubmission, string driverFolder) { var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray()); var bootstrapArgs = new AvroJobSubmissionParameters { jobSubmissionFolder = driverFolder, jobId = jobSubmission.JobIdentifier, tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(), tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(), tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(), }; var avroLocalBootstrapArgs = new AvroLocalJobSubmissionParameters { sharedJobSubmissionParameters = bootstrapArgs, maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators }; var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { var serializedArgs = AvroJsonSerializer <AvroLocalJobSubmissionParameters> .ToBytes(avroLocalBootstrapArgs); argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } return(submissionArgsFilePath); }
/// <summary> /// Initializes a new instance of the <see cref="AvroYarnJobSubmissionParameters"/> class. /// </summary> /// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param> /// <param name="dfsJobSubmissionFolder">The dfsJobSubmissionFolder.</param> /// <param name="jobSubmissionDirectoryPrefix">The jobSubmissionDirectoryPrefix.</param> /// <param name="fileSystemUrl">The file system URL prefix.</param> public AvroYarnJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, string dfsJobSubmissionFolder, string fileSystemUrl, string jobSubmissionDirectoryPrefix) { this.sharedJobSubmissionParameters = sharedJobSubmissionParameters; this.dfsJobSubmissionFolder = dfsJobSubmissionFolder; this.fileSystemUrl = fileSystemUrl; this.jobSubmissionDirectoryPrefix = jobSubmissionDirectoryPrefix; }
/// <summary> /// Initializes a new instance of the <see cref="AvroYarnJobSubmissionParameters"/> class. /// </summary> /// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param> /// <param name="driverMemory">The driverMemory.</param> /// <param name="driverRecoveryTimeout">The driverRecoveryTimeout.</param> /// <param name="dfsJobSubmissionFolder">The dfsJobSubmissionFolder.</param> /// <param name="jobSubmissionDirectoryPrefix">The jobSubmissionDirectoryPrefix.</param> public AvroYarnJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, int driverMemory, int driverRecoveryTimeout, string dfsJobSubmissionFolder, string jobSubmissionDirectoryPrefix) { this.sharedJobSubmissionParameters = sharedJobSubmissionParameters; this.driverMemory = driverMemory; this.driverRecoveryTimeout = driverRecoveryTimeout; this.dfsJobSubmissionFolder = dfsJobSubmissionFolder; this.jobSubmissionDirectoryPrefix = jobSubmissionDirectoryPrefix; }
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string localDriverFolderPath, string jobSubmissionDirectory) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = localDriverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { jobSubmissionDirectoryPrefix = jobSubmissionDirectory, dfsJobSubmissionFolder = jobSubmissionDirectory, sharedJobSubmissionParameters = avroJobSubmissionParameters }; return(AvroJsonSerializer <AvroYarnJobSubmissionParameters> .ToBytes(avroYarnJobSubmissionParameters)); }
private void Launch(IJobSubmission jobSubmission, string driverFolderPath) { _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath); // TODO: Remove this when we have a generalized way to pass config to java var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray()); var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobSubmission.JobIdentifier, tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(), tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(), tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(), jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { driverMemory = jobSubmission.DriverMemory, driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(), jobSubmissionDirectoryPrefix = _jobSubmissionPrefix, sharedJobSubmissionParameters = avroJobSubmissionParameters }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { maxApplicationSubmissions = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(), securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters }; var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { var serializedArgs = AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters); argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } // Submit the driver _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath); Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier); }
/// <summary> /// Initializes a new instance of the <see cref="AvroYarnJobSubmissionParameters"/> class. /// </summary> /// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param> /// <param name="driverMemory">The driverMemory.</param> /// <param name="driverRecoveryTimeout">The driverRecoveryTimeout.</param> /// <param name="dfsJobSubmissionFolder">The dfsJobSubmissionFolder.</param> /// <param name="jobSubmissionDirectoryPrefix">The jobSubmissionDirectoryPrefix.</param> public AvroYarnJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, int driverMemory, int driverRecoveryTimeout, string dfsJobSubmissionFolder, string jobSubmissionDirectoryPrefix) { this.sharedJobSubmissionParameters = sharedJobSubmissionParameters; this.driverMemory = driverMemory; this.driverRecoveryTimeout = driverRecoveryTimeout; this.dfsJobSubmissionFolder = dfsJobSubmissionFolder; this.jobSubmissionDirectoryPrefix = jobSubmissionDirectoryPrefix; }
/// <summary> /// Initializes a new instance of the <see cref="AvroLocalJobSubmissionParameters"/> class. /// </summary> /// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param> /// <param name="maxNumberOfConcurrentEvaluators">The maxNumberOfConcurrentEvaluators.</param> public AvroLocalJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, int maxNumberOfConcurrentEvaluators) { this.sharedJobSubmissionParameters = sharedJobSubmissionParameters; this.maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators; }
/// <summary> /// Initializes a new instance of the <see cref="AvroYarnJobSubmissionParameters"/> class. /// </summary> /// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param> /// <param name="dfsJobSubmissionFolder">The dfsJobSubmissionFolder.</param> /// <param name="jobSubmissionDirectoryPrefix">The jobSubmissionDirectoryPrefix.</param> public AvroYarnJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, string dfsJobSubmissionFolder, string jobSubmissionDirectoryPrefix) { this.sharedJobSubmissionParameters = sharedJobSubmissionParameters; this.dfsJobSubmissionFolder = dfsJobSubmissionFolder; this.jobSubmissionDirectoryPrefix = jobSubmissionDirectoryPrefix; }
/// <summary> /// Initializes a new instance of the <see cref="AvroLocalJobSubmissionParameters"/> class. /// </summary> /// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param> /// <param name="maxNumberOfConcurrentEvaluators">The maxNumberOfConcurrentEvaluators.</param> public AvroLocalJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, int maxNumberOfConcurrentEvaluators) { this.sharedJobSubmissionParameters = sharedJobSubmissionParameters; this.maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators; }
/// <summary> /// Initializes a new instance of the <see cref="AvroLocalJobSubmissionParameters"/> class. /// </summary> /// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param> /// <param name="driverStdoutFilePath">The driverStdoutFilePath.</param> /// <param name="driverStderrFilePath">The driverStderrFilePath.</param> public AvroLocalJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, string driverStdoutFilePath, string driverStderrFilePath) { this.sharedJobSubmissionParameters = sharedJobSubmissionParameters; this.driverStdoutFilePath = driverStdoutFilePath; this.driverStderrFilePath = driverStderrFilePath; }
/// <summary> /// Initializes a new instance of the <see cref="AvroLocalJobSubmissionParameters"/> class. /// </summary> /// <param name="sharedJobSubmissionParameters">The sharedJobSubmissionParameters.</param> /// <param name="driverStdoutFilePath">The driverStdoutFilePath.</param> /// <param name="driverStderrFilePath">The driverStderrFilePath.</param> public AvroLocalJobSubmissionParameters(AvroJobSubmissionParameters sharedJobSubmissionParameters, string driverStdoutFilePath, string driverStderrFilePath) { this.sharedJobSubmissionParameters = sharedJobSubmissionParameters; this.driverStdoutFilePath = driverStdoutFilePath; this.driverStderrFilePath = driverStderrFilePath; }