private string CreateBootstrapAvroJobConfig(JobParameters jobParameters, string driverFolder)
{
var bootstrapJobArgs = new AvroJobSubmissionParameters
{
jobSubmissionFolder = driverFolder,
jobId = jobParameters.JobIdentifier,
};
var bootstrapLocalJobArgs = new AvroLocalJobSubmissionParameters
{
sharedJobSubmissionParameters = bootstrapJobArgs,
driverStdoutFilePath = jobParameters.StdoutFilePath.IsPresent() ? jobParameters.StdoutFilePath.Value : _fileNames.GetDriverStdoutFileName(),
driverStderrFilePath = jobParameters.StderrFilePath.IsPresent() ? jobParameters.StderrFilePath.Value : _fileNames.GetDriverStderrFileName()
};
var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroLocalJobSubmissionParameters> .ToBytes(bootstrapLocalJobArgs);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return(submissionArgsFilePath);
}
private string CreateBootstrapAvroAppConfig(AppParameters appParameters, string driverFolder)
{
var paramInjector = TangFactory.GetTang().NewInjector(appParameters.DriverConfigurations.ToArray());
var bootstrapAppArgs = new AvroAppSubmissionParameters
{
tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(),
};
var avroLocalBootstrapAppArgs = new AvroLocalAppSubmissionParameters
{
sharedAppSubmissionParameters = bootstrapAppArgs,
maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators
};
var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetAppSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroLocalAppSubmissionParameters> .ToBytes(avroLocalBootstrapAppArgs);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return(submissionArgsFilePath);
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
driverMemory = jobParameters.DriverMemoryInMB,
maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions,
driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value,
driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value
};
return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters));
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
driverMemory = jobParameters.DriverMemoryInMB,
maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions
};
return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters));
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
sharedJobSubmissionParameters = avroJobSubmissionParameters,
fileSystemUrl = _fileSystemUrl,
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
// TODO[JIRA REEF-1887] Deprecated. Remove in REEF 0.18.
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
driverMemory = jobParameters.DriverMemoryInMB,
environmentVariablesMap = jobParameters.JobSubmissionEnvMap,
maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions,
driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value,
driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value
};
return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters));
}
/// <summary>
/// THIS ASSUMES THAT IT IS CALLED ON A THREAD HOLDING THE LOCK ON THE HeartBeatManager
/// </summary>
/// <param name="e"></param>
private void HandleTaskInitializationException(TaskClientCodeException e)
{
byte[] error;
try
{
error = ByteUtilities.SerializeToBinaryFormat(e);
}
catch (SerializationException se)
{
error = ByteUtilities.SerializeToBinaryFormat(
TaskClientCodeException.CreateWithNonSerializableInnerException(e, se));
}
var avroFailedTask = new AvroFailedTask
{
identifier = e.TaskId,
cause = error,
data = ByteUtilities.StringToByteArrays(e.ToString()),
message = e.Message
};
var taskStatus = new TaskStatusProto
{
context_id = e.ContextId,
task_id = e.TaskId,
result = AvroJsonSerializer <AvroFailedTask> .ToBytes(avroFailedTask),
state = State.FAILED
};
LOGGER.Log(Level.Error, "Sending Heartbeat for a failed task: {0}", taskStatus);
_heartBeatManager.OnNext(taskStatus);
}
public TaskStatusProto ToProto()
{
// This is locked because the Task continuation thread which sets the
// result is potentially different from the HeartBeat thread.
lock (_heartBeatManager)
{
Check();
TaskStatusProto taskStatusProto = new TaskStatusProto()
{
context_id = ContextId,
task_id = TaskId,
state = GetProtoState()
};
if (_result.IsPresent())
{
taskStatusProto.result = ByteUtilities.CopyBytesFrom(_result.Value);
}
else if (_lastException.IsPresent())
{
byte[] error;
try
{
error = ByteUtilities.SerializeToBinaryFormat(_lastException.Value);
}
catch (SerializationException se)
{
error = ByteUtilities.SerializeToBinaryFormat(
NonSerializableTaskException.UnableToSerialize(_lastException.Value, se));
}
var avroFailedTask = new AvroFailedTask
{
identifier = _taskId,
cause = error,
data = ByteUtilities.StringToByteArrays(_lastException.Value.ToString()),
message = _lastException.Value.Message
};
taskStatusProto.result = AvroJsonSerializer <AvroFailedTask> .ToBytes(avroFailedTask);
}
else if (_state == TaskState.Running)
{
foreach (TaskMessage message in GetMessages())
{
TaskStatusProto.TaskMessageProto taskMessageProto = new TaskStatusProto.TaskMessageProto()
{
source_id = message.MessageSourceId,
message = ByteUtilities.CopyBytesFrom(message.Message),
};
taskStatusProto.task_message.Add(taskMessageProto);
}
}
return(taskStatusProto);
}
}
private void SerializeJobFile(string localDriverFolderPath, AvroAzureBatchJobSubmissionParameters jobParameters)
{
var serializedArgs = AvroJsonSerializer <AvroAzureBatchJobSubmissionParameters> .ToBytes(jobParameters);
var submissionJobArgsFilePath = Path.Combine(localDriverFolderPath,
_fileNames.GetReefFolderName(), _fileNames.GetJobSubmissionParametersFile());
using (var jobArgsFileStream = new FileStream(submissionJobArgsFilePath, FileMode.CreateNew))
{
jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
}
internal byte[] SerializeAppArgsToBytes(AppParameters appParameters, IInjector paramInjector)
{
var avroAppSubmissionParameters = new AvroAppSubmissionParameters
{
tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>()
};
var avroYarnAppSubmissionParameters = new AvroYarnAppSubmissionParameters
{
sharedAppSubmissionParameters = avroAppSubmissionParameters,
driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>()
};
return(AvroJsonSerializer <AvroYarnAppSubmissionParameters> .ToBytes(avroYarnAppSubmissionParameters));
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string localDriverFolderPath, string jobSubmissionDirectory)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = localDriverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = jobSubmissionDirectory,
dfsJobSubmissionFolder = jobSubmissionDirectory,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
return(AvroJsonSerializer <AvroYarnJobSubmissionParameters> .ToBytes(avroYarnJobSubmissionParameters));
}
private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
{
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);
// TODO: Remove this when we have a generalized way to pass config to java
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(),
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
driverMemory = jobSubmission.DriverMemory,
driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
maxApplicationSubmissions = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(),
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters
};
var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
// Submit the driver
_javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
}
