private string CreateBootstrapAvroJobConfig(JobParameters jobParameters, string driverFolder) { var bootstrapJobArgs = new AvroJobSubmissionParameters { jobSubmissionFolder = driverFolder, jobId = jobParameters.JobIdentifier, }; var bootstrapLocalJobArgs = new AvroLocalJobSubmissionParameters { sharedJobSubmissionParameters = bootstrapJobArgs, driverStdoutFilePath = jobParameters.StdoutFilePath.IsPresent() ? jobParameters.StdoutFilePath.Value : _fileNames.GetDriverStdoutFileName(), driverStderrFilePath = jobParameters.StderrFilePath.IsPresent() ? jobParameters.StderrFilePath.Value : _fileNames.GetDriverStderrFileName() }; var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { var serializedArgs = AvroJsonSerializer <AvroLocalJobSubmissionParameters> .ToBytes(bootstrapLocalJobArgs); argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } return(submissionArgsFilePath); }
private string CreateBootstrapAvroAppConfig(AppParameters appParameters, string driverFolder) { var paramInjector = TangFactory.GetTang().NewInjector(appParameters.DriverConfigurations.ToArray()); var bootstrapAppArgs = new AvroAppSubmissionParameters { tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(), tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(), tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(), }; var avroLocalBootstrapAppArgs = new AvroLocalAppSubmissionParameters { sharedAppSubmissionParameters = bootstrapAppArgs, maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators }; var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetAppSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { var serializedArgs = AvroJsonSerializer <AvroLocalAppSubmissionParameters> .ToBytes(avroLocalBootstrapAppArgs); argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } return(submissionArgsFilePath); }
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { jobSubmissionDirectoryPrefix = _jobSubmissionPrefix, sharedJobSubmissionParameters = avroJobSubmissionParameters }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, driverMemory = jobParameters.DriverMemoryInMB, maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions, driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ? _fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value, driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ? _fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value }; return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters)); }
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { jobSubmissionDirectoryPrefix = _jobSubmissionPrefix, sharedJobSubmissionParameters = avroJobSubmissionParameters }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, driverMemory = jobParameters.DriverMemoryInMB, maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions }; return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters)); }
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { sharedJobSubmissionParameters = avroJobSubmissionParameters, fileSystemUrl = _fileSystemUrl, jobSubmissionDirectoryPrefix = _jobSubmissionPrefix }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { // TODO[JIRA REEF-1887] Deprecated. Remove in REEF 0.18. securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters, driverMemory = jobParameters.DriverMemoryInMB, environmentVariablesMap = jobParameters.JobSubmissionEnvMap, maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions, driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ? _fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value, driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ? _fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value }; return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters)); }
/// <summary> /// THIS ASSUMES THAT IT IS CALLED ON A THREAD HOLDING THE LOCK ON THE HeartBeatManager /// </summary> /// <param name="e"></param> private void HandleTaskInitializationException(TaskClientCodeException e) { byte[] error; try { error = ByteUtilities.SerializeToBinaryFormat(e); } catch (SerializationException se) { error = ByteUtilities.SerializeToBinaryFormat( TaskClientCodeException.CreateWithNonSerializableInnerException(e, se)); } var avroFailedTask = new AvroFailedTask { identifier = e.TaskId, cause = error, data = ByteUtilities.StringToByteArrays(e.ToString()), message = e.Message }; var taskStatus = new TaskStatusProto { context_id = e.ContextId, task_id = e.TaskId, result = AvroJsonSerializer <AvroFailedTask> .ToBytes(avroFailedTask), state = State.FAILED }; LOGGER.Log(Level.Error, "Sending Heartbeat for a failed task: {0}", taskStatus); _heartBeatManager.OnNext(taskStatus); }
public TaskStatusProto ToProto() { // This is locked because the Task continuation thread which sets the // result is potentially different from the HeartBeat thread. lock (_heartBeatManager) { Check(); TaskStatusProto taskStatusProto = new TaskStatusProto() { context_id = ContextId, task_id = TaskId, state = GetProtoState() }; if (_result.IsPresent()) { taskStatusProto.result = ByteUtilities.CopyBytesFrom(_result.Value); } else if (_lastException.IsPresent()) { byte[] error; try { error = ByteUtilities.SerializeToBinaryFormat(_lastException.Value); } catch (SerializationException se) { error = ByteUtilities.SerializeToBinaryFormat( NonSerializableTaskException.UnableToSerialize(_lastException.Value, se)); } var avroFailedTask = new AvroFailedTask { identifier = _taskId, cause = error, data = ByteUtilities.StringToByteArrays(_lastException.Value.ToString()), message = _lastException.Value.Message }; taskStatusProto.result = AvroJsonSerializer <AvroFailedTask> .ToBytes(avroFailedTask); } else if (_state == TaskState.Running) { foreach (TaskMessage message in GetMessages()) { TaskStatusProto.TaskMessageProto taskMessageProto = new TaskStatusProto.TaskMessageProto() { source_id = message.MessageSourceId, message = ByteUtilities.CopyBytesFrom(message.Message), }; taskStatusProto.task_message.Add(taskMessageProto); } } return(taskStatusProto); } }
private void SerializeJobFile(string localDriverFolderPath, AvroAzureBatchJobSubmissionParameters jobParameters) { var serializedArgs = AvroJsonSerializer <AvroAzureBatchJobSubmissionParameters> .ToBytes(jobParameters); var submissionJobArgsFilePath = Path.Combine(localDriverFolderPath, _fileNames.GetReefFolderName(), _fileNames.GetJobSubmissionParametersFile()); using (var jobArgsFileStream = new FileStream(submissionJobArgsFilePath, FileMode.CreateNew)) { jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } }
internal byte[] SerializeAppArgsToBytes(AppParameters appParameters, IInjector paramInjector) { var avroAppSubmissionParameters = new AvroAppSubmissionParameters { tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(), tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(), tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>() }; var avroYarnAppSubmissionParameters = new AvroYarnAppSubmissionParameters { sharedAppSubmissionParameters = avroAppSubmissionParameters, driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>() }; return(AvroJsonSerializer <AvroYarnAppSubmissionParameters> .ToBytes(avroYarnAppSubmissionParameters)); }
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string localDriverFolderPath, string jobSubmissionDirectory) { var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobParameters.JobIdentifier, jobSubmissionFolder = localDriverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { jobSubmissionDirectoryPrefix = jobSubmissionDirectory, dfsJobSubmissionFolder = jobSubmissionDirectory, sharedJobSubmissionParameters = avroJobSubmissionParameters }; return(AvroJsonSerializer <AvroYarnJobSubmissionParameters> .ToBytes(avroYarnJobSubmissionParameters)); }
private void Launch(IJobSubmission jobSubmission, string driverFolderPath) { _driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath); // TODO: Remove this when we have a generalized way to pass config to java var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray()); var avroJobSubmissionParameters = new AvroJobSubmissionParameters { jobId = jobSubmission.JobIdentifier, tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(), tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(), tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(), jobSubmissionFolder = driverFolderPath }; var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters { driverMemory = jobSubmission.DriverMemory, driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(), jobSubmissionDirectoryPrefix = _jobSubmissionPrefix, sharedJobSubmissionParameters = avroJobSubmissionParameters }; var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters { maxApplicationSubmissions = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(), securityTokenKind = _securityTokenKind, securityTokenService = _securityTokenService, yarnJobSubmissionParameters = avroYarnJobSubmissionParameters }; var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile()); using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew)) { var serializedArgs = AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters); argsFileStream.Write(serializedArgs, 0, serializedArgs.Length); } // Submit the driver _javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath); Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier); }