private string CreateBootstrapAvroJobConfig(JobParameters jobParameters, string driverFolder)
{
var bootstrapJobArgs = new AvroJobSubmissionParameters
{
jobSubmissionFolder = driverFolder,
jobId = jobParameters.JobIdentifier,
};
var bootstrapLocalJobArgs = new AvroLocalJobSubmissionParameters
{
sharedJobSubmissionParameters = bootstrapJobArgs,
driverStdoutFilePath = jobParameters.StdoutFilePath.IsPresent() ? jobParameters.StdoutFilePath.Value : _fileNames.GetDriverStdoutFileName(),
driverStderrFilePath = jobParameters.StderrFilePath.IsPresent() ? jobParameters.StderrFilePath.Value : _fileNames.GetDriverStderrFileName()
};
var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroLocalJobSubmissionParameters> .ToBytes(bootstrapLocalJobArgs);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return(submissionArgsFilePath);
}
private string CreateBootstrapAvroAppConfig(AppParameters appParameters, string driverFolder)
{
var paramInjector = TangFactory.GetTang().NewInjector(appParameters.DriverConfigurations.ToArray());
var bootstrapAppArgs = new AvroAppSubmissionParameters
{
tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(),
};
var avroLocalBootstrapAppArgs = new AvroLocalAppSubmissionParameters
{
sharedAppSubmissionParameters = bootstrapAppArgs,
maxNumberOfConcurrentEvaluators = _maxNumberOfConcurrentEvaluators
};
var submissionArgsFilePath = Path.Combine(driverFolder, _fileNames.GetAppSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroLocalAppSubmissionParameters> .ToBytes(avroLocalBootstrapAppArgs);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
return(submissionArgsFilePath);
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
driverMemory = jobParameters.DriverMemoryInMB,
maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions
};
return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters));
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
driverMemory = jobParameters.DriverMemoryInMB,
maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions,
driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value,
driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value
};
return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters));
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string driverFolderPath)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
sharedJobSubmissionParameters = avroJobSubmissionParameters,
fileSystemUrl = _fileSystemUrl,
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
// TODO[JIRA REEF-1887] Deprecated. Remove in REEF 0.18.
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters,
driverMemory = jobParameters.DriverMemoryInMB,
environmentVariablesMap = jobParameters.JobSubmissionEnvMap,
maxApplicationSubmissions = jobParameters.MaxApplicationSubmissions,
driverStdoutFilePath = string.IsNullOrWhiteSpace(jobParameters.StdoutFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStdoutFilePath() : jobParameters.StdoutFilePath.Value,
driverStderrFilePath = string.IsNullOrWhiteSpace(jobParameters.StderrFilePath.Value) ?
_fileNames.GetDefaultYarnDriverStderrFilePath() : jobParameters.StderrFilePath.Value
};
return(AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters));
}
/// <summary>
/// THIS ASSUMES THAT IT IS CALLED ON A THREAD HOLDING THE LOCK ON THE HeartBeatManager
/// </summary>
/// <param name="e"></param>
private void HandleTaskInitializationException(TaskClientCodeException e)
{
byte[] error;
try
{
error = ByteUtilities.SerializeToBinaryFormat(e);
}
catch (SerializationException se)
{
error = ByteUtilities.SerializeToBinaryFormat(
TaskClientCodeException.CreateWithNonSerializableInnerException(e, se));
}
var avroFailedTask = new AvroFailedTask
{
identifier = e.TaskId,
cause = error,
data = ByteUtilities.StringToByteArrays(e.ToString()),
message = e.Message
};
var taskStatus = new TaskStatusProto
{
context_id = e.ContextId,
task_id = e.TaskId,
result = AvroJsonSerializer <AvroFailedTask> .ToBytes(avroFailedTask),
state = State.FAILED
};
LOGGER.Log(Level.Error, "Sending Heartbeat for a failed task: {0}", taskStatus);
_heartBeatManager.OnNext(taskStatus);
}
public void CanExtractDriverInformaiton()
{
const string InfoString = "{\"remoteId\":\"socket://10.121.136.231:14272\",\"startTime\":\"2014 08 28 10:50:32\",\"services\":[{\"serviceName\":\"NameServer\",\"serviceInfo\":\"10.121.136.231:16663\"}]}";
AvroDriverInfo info = AvroJsonSerializer <AvroDriverInfo> .FromString(InfoString);
Assert.IsTrue(info.remoteId.Equals("socket://10.121.136.231:14272"));
Assert.IsTrue(info.startTime.Equals("2014 08 28 10:50:32"));
Assert.IsTrue(new DriverInformation(info.remoteId, info.startTime, info.services).NameServerId.Equals("10.121.136.231:16663"));
}
public TaskStatusProto ToProto()
{
// This is locked because the Task continuation thread which sets the
// result is potentially different from the HeartBeat thread.
lock (_heartBeatManager)
{
Check();
TaskStatusProto taskStatusProto = new TaskStatusProto()
{
context_id = ContextId,
task_id = TaskId,
state = GetProtoState()
};
if (_result.IsPresent())
{
taskStatusProto.result = ByteUtilities.CopyBytesFrom(_result.Value);
}
else if (_lastException.IsPresent())
{
byte[] error;
try
{
error = ByteUtilities.SerializeToBinaryFormat(_lastException.Value);
}
catch (SerializationException se)
{
error = ByteUtilities.SerializeToBinaryFormat(
NonSerializableTaskException.UnableToSerialize(_lastException.Value, se));
}
var avroFailedTask = new AvroFailedTask
{
identifier = _taskId,
cause = error,
data = ByteUtilities.StringToByteArrays(_lastException.Value.ToString()),
message = _lastException.Value.Message
};
taskStatusProto.result = AvroJsonSerializer <AvroFailedTask> .ToBytes(avroFailedTask);
}
else if (_state == TaskState.Running)
{
foreach (TaskMessage message in GetMessages())
{
TaskStatusProto.TaskMessageProto taskMessageProto = new TaskStatusProto.TaskMessageProto()
{
source_id = message.MessageSourceId,
message = ByteUtilities.CopyBytesFrom(message.Message),
};
taskStatusProto.task_message.Add(taskMessageProto);
}
}
return(taskStatusProto);
}
}
private void SerializeJobFile(string localDriverFolderPath, AvroAzureBatchJobSubmissionParameters jobParameters)
{
var serializedArgs = AvroJsonSerializer <AvroAzureBatchJobSubmissionParameters> .ToBytes(jobParameters);
var submissionJobArgsFilePath = Path.Combine(localDriverFolderPath,
_fileNames.GetReefFolderName(), _fileNames.GetJobSubmissionParametersFile());
using (var jobArgsFileStream = new FileStream(submissionJobArgsFilePath, FileMode.CreateNew))
{
jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
}
public static DriverInformation GetDriverInformationFromHttp(Uri queryUri)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(queryUri);
request.AllowAutoRedirect = false;
request.KeepAlive = false;
request.ContentType = "text/html";
string driverInfomation;
AvroDriverInfo info = null;
try
{
using (HttpWebResponse webResponse = (HttpWebResponse)request.GetResponse())
{
Stream stream = webResponse.GetResponseStream();
if (stream == null)
{
return(null);
}
using (StreamReader streamReader = new StreamReader(stream, Encoding.UTF8))
{
driverInfomation = streamReader.ReadToEnd();
LOGGER.Log(Level.Verbose, "Http response line: " + driverInfomation);
info = AvroJsonSerializer <AvroDriverInfo> .FromString(driverInfomation);
}
}
}
catch (WebException)
{
LOGGER.Log(Level.Warning, string.Format(CultureInfo.InvariantCulture, "In RECOVERY mode, cannot connect to [{0}] for driver information, will try again later.", queryUri));
return(null);
}
catch (Exception e)
{
Org.Apache.Reef.Utilities.Diagnostics.Exceptions.CaughtAndThrow(e, Level.Error, string.Format(CultureInfo.InvariantCulture, "Cannot read content from {0}.", queryUri), LOGGER);
}
if (info != null)
{
LOGGER.Log(
Level.Verbose,
string.Format(CultureInfo.InvariantCulture, "Driver information extracted with remote identier [{0}], start time [{1}], and servics [{2}]", info.remoteId, info.startTime, info.services));
return(new DriverInformation(info.remoteId, info.startTime, info.services));
}
return(null);
}
internal byte[] SerializeAppArgsToBytes(AppParameters appParameters, IInjector paramInjector)
{
var avroAppSubmissionParameters = new AvroAppSubmissionParameters
{
tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>()
};
var avroYarnAppSubmissionParameters = new AvroYarnAppSubmissionParameters
{
sharedAppSubmissionParameters = avroAppSubmissionParameters,
driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>()
};
return(AvroJsonSerializer <AvroYarnAppSubmissionParameters> .ToBytes(avroYarnAppSubmissionParameters));
}
internal byte[] SerializeJobArgsToBytes(JobParameters jobParameters, string localDriverFolderPath, string jobSubmissionDirectory)
{
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobParameters.JobIdentifier,
jobSubmissionFolder = localDriverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
jobSubmissionDirectoryPrefix = jobSubmissionDirectory,
dfsJobSubmissionFolder = jobSubmissionDirectory,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
return(AvroJsonSerializer <AvroYarnJobSubmissionParameters> .ToBytes(avroYarnJobSubmissionParameters));
}
private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
{
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);
// TODO: Remove this when we have a generalized way to pass config to java
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(),
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
driverMemory = jobSubmission.DriverMemory,
driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
maxApplicationSubmissions = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(),
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters
};
var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
// Submit the driver
_javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
}
public BridgeFailedTask(
Optional <IActiveContext> activeContext,
string id,
string message,
string description,
string reason,
byte[] data)
{
_activeContext = activeContext;
Id = id;
Message = message;
Description = Optional <string> .OfNullable(description);
Reason = Optional <string> .OfNullable(reason);
if (data != null)
{
var avroFailedTask = AvroJsonSerializer <AvroFailedTask> .FromBytes(data);
if (!id.Equals(avroFailedTask.identifier))
{
Log.Log(Level.Error, "Task error id {0} does not match task id {1}", avroFailedTask.identifier, id);
}
// Data is simply the serialized Exception.ToString.
Data = Optional <byte[]> .OfNullable(avroFailedTask.data);
// Message can be overwritten in Java, if the C# Message is null and the Task failure is caused by an Evaluator failure.
Message = string.IsNullOrWhiteSpace(avroFailedTask.message)
? "No message in Failed Task."
: avroFailedTask.message;
// Gets the Exception.
_cause = GetCause(avroFailedTask.cause, ByteUtilities.ByteArraysToString(avroFailedTask.data));
// This is always empty, even in Java.
Description = Optional <string> .Empty();
}
else
{
Log.Log(Level.Warning, "no exception data");
Data = Optional <byte[]> .Empty();
}
}
public FailedTask(IFailedTaskClr2Java failedTaskClr2Java)
{
var serializedInfo = failedTaskClr2Java.GetFailedTaskSerializedAvro();
var avroFailedTask = AvroJsonSerializer <AvroFailedTask> .FromBytes(serializedInfo);
Id = avroFailedTask.identifier;
// Data is simply the serialized Exception.ToString.
Data = Optional <byte[]> .OfNullable(avroFailedTask.data);
// Message can be overwritten in Java, if the C# Message is null and the Task failure is caused by an Evaluator failure.
Message = string.IsNullOrWhiteSpace(avroFailedTask.message) ? "No message in Failed Task." : avroFailedTask.message;
// Gets the Exception.
_cause = GetCause(avroFailedTask.cause, ByteUtilities.ByteArraysToString(avroFailedTask.data));
// This is always empty, even in Java.
Description = Optional <string> .Empty();
ActiveContextClr2Java = failedTaskClr2Java.GetActiveContext();
}
public static DriverInformation GetDriverInformationFromHttp(Uri queryUri)
{
while (queryUri != null)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(queryUri);
request.AllowAutoRedirect = true;
request.KeepAlive = false;
request.ContentType = "text/html";
queryUri = null;
try
{
using (HttpWebResponse webResponse = (HttpWebResponse)request.GetResponse())
{
var refresh = webResponse.Headers.AllKeys.FirstOrDefault(k => k.Equals("refresh", StringComparison.OrdinalIgnoreCase));
if (refresh != null)
{
var refreshContent = webResponse.Headers.GetValues(refresh);
foreach (var refreshParam in refreshContent.SelectMany(content => content.Split(';').Select(c => c.Trim())))
{
var refreshKeyValue = refreshParam.Split('=').Select(kv => kv.Trim()).ToArray();
if (refreshKeyValue.Length == 2 && refreshKeyValue[0].Equals("url", StringComparison.OrdinalIgnoreCase))
{
queryUri = new Uri(refreshKeyValue[1]);
break;
}
}
}
// We have received a redirect URI, look there instead.
if (queryUri != null)
{
LOGGER.Log(Level.Verbose, "Received redirect URI:[{0}], redirecting...", queryUri);
continue;
}
Stream stream = webResponse.GetResponseStream();
if (stream == null || !stream.CanRead)
{
return(null);
}
using (StreamReader streamReader = new StreamReader(stream, Encoding.UTF8))
{
var driverInformation = streamReader.ReadToEnd();
LOGGER.Log(Level.Verbose, "Http response line: {0}", driverInformation);
AvroDriverInfo info = null;
try
{
info = AvroJsonSerializer <AvroDriverInfo> .FromString(driverInformation);
}
catch (Exception e)
{
Utilities.Diagnostics.Exceptions.CaughtAndThrow(
e, Level.Error, string.Format(CultureInfo.InvariantCulture, "Cannot read content: {0}.", driverInformation), LOGGER);
}
if (info == null)
{
LOGGER.Log(Level.Info, "Cannot read content: {0}.", driverInformation);
return(null);
}
return(new DriverInformation(info.remoteId, info.startTime, info.services));
}
}
}
catch (WebException)
{
LOGGER.Log(Level.Warning, "In RECOVERY mode, cannot connect to [{0}] for driver information, will try again later.", queryUri);
return(null);
}
}
return(null);
}