/// <summary>
/// Make sure all pending properties get handled and vertex complete event is sent to GM
/// </summary>
/// <param name="obj"></param>
void ExitProcessThreadProc(Object obj)
{
DryadLogger.LogMethodEntry();
//
// Wait until all property waiters have been notified and the final
// status message sent, iff the process completed successfully
//
do
{
//
// Clear any thing intended for the vertex
//
SetAllPropertyWaiters();
lock (syncRoot)
{
// If nobody is waiting, AND
if (propertyWaiters == 0)
{
// Process did not complete successfully OR
// final status message has already been sent
if (!Succeeded || finalStatusMessageSent)
{
// Then we can send the Process Exit notification
break;
}
}
}
Thread.Sleep(10);
} while(true);
ReplyDispatcher.ProcessExited(this.graphManagerReplyUri, this.dryadProcessId, this.exitCode);
//
// This should never happen unless a property is requested after the vertex completed event is sent
// so it's not a big deal if it does because the GM knows that the vertex is done
//
if (propertyWaiters > 0)
{
DryadLogger.LogWarning("Process exit", "Leaving thread with {0} property waiter(s).", propertyWaiters);
}
DryadLogger.LogMethodExit();
}
/// <summary>
/// Creates a request for property update from one vertex to another (one vertex is usually GM)
/// </summary>
/// <param name="replyEpr">Address to send response to</param>
/// <param name="infos"></param>
/// <param name="blockOnLabel"></param>
/// <param name="blockOnVersion"></param>
/// <param name="maxBlockTime"></param>
/// <param name="getPropLabel"></param>
/// <param name="ProcessStatistics"></param>
/// <returns>Returns success/failure of thread startup</returns>
public bool SetGetProps(string replyEpr, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics)
{
DryadLogger.LogMethodEntry(replyEpr, this.DryadId);
//
// Check if graph manager is the one calling. If so, increment propertyWaiter count
//
if (ReplyDispatcher.IsGraphMrgUri(replyEpr))
{
int n = Interlocked.Increment(ref propertyWaiters);
}
//
// Create new property request with provided parameters and queue up request sending
//
PropertyRequest req = new PropertyRequest(replyEpr, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics);
bool result = ThreadPool.QueueUserWorkItem(new WaitCallback(SetGetPropThreadProc), req);
DryadLogger.LogMethodExit(result);
return(result);
}
/// <summary>
/// Called in new thread in setgetproperty service operation
/// </summary>
/// <param name="obj"></param>
void SetGetPropThreadProc(Object obj)
{
DryadLogger.LogMethodEntry(DryadId);
PropertyRequest r = obj as PropertyRequest;
ProcessInfo infoLocal = new ProcessInfo();
ulong[] propertyVersions = null;
string[] propertyLabels = null;
//
// Make sure process is started before continuing
//
if (this.State < ProcessState.Running)
{
try
{
processStartEvent.WaitOne();
}
catch (ObjectDisposedException ex)
{
// The process was cancelled and released before it started running, just return
if (exited)
{
DryadLogger.LogInformation("SetGetProp Thread", "Process {0} cancelled or exited before starting.", this.DryadId);
}
else
{
DryadLogger.LogError(0, ex);
}
DryadLogger.LogMethodExit();
return;
}
}
//
// Use status_pending if running, vertex initialization failure if process is failed and process exit code otherwise
//
infoLocal.processStatus = 0x103; // WinNT.h STATUS_PENDING
infoLocal.processState = state;
if (state == ProcessState.Running)
{
infoLocal.exitCode = 0x103; // WinNT.h STATUS_PENDING
}
else if (failed)
{
infoLocal.exitCode = Constants.DrError_VertexError;
}
else if (cancelled)
{
infoLocal.exitCode = Constants.DrError_VertexReceivedTermination; // DryadError_VertexReceivedTermination
}
else
{
infoLocal.exitCode = (uint)systemProcess.ExitCode;
}
//
// Record specified properties and update versions - wakes up anyone waiting for property changes
//
SetProperties(r.infos, out propertyLabels, out propertyVersions);
//
// Try to get property update
//
if (BlockOnProperty(r.blockOnLabel, r.blockOnVersion, r.maxBlockTime))
{
//
// If property update was received, update the received property information
// If received property marks vertex completed, record that
//
if (r.getPropLabel != null && r.getPropLabel.Length > 0)
{
lock (syncRoot)
{
infoLocal.propertyInfos = new ProcessPropertyInfo[1];
int index;
if (TryGetProperty(r.getPropLabel, out infoLocal.propertyInfos[0], out index) == false)
{
DryadLogger.LogError(0, null, "Failed to get property for label {0}", r.getPropLabel);
}
if (StatusMessageContainsDryadError_VertexCompleted(infoLocal.propertyInfos[0].propertyLabel))
{
CopyProp(infoLocal.propertyInfos[0], out latestVertexStatusSent);
}
}
}
//
// If request asks for statistics on vertex process, get them
//
if (r.ProcessStatistics)
{
if (GetStatistics(out infoLocal.processStatistics) == false)
{
DryadLogger.LogError(0, null, "Failed to get vertex statistics");
}
}
}
//
// Try to report property change, if unsuccessful, kill the running vertex host process
//
if (!ReplyDispatcher.SetGetPropsComplete(r.replyUri, systemProcess, dryadProcessId, infoLocal, propertyLabels, propertyVersions))
{
try
{
systemProcess.Kill();
}
catch (InvalidOperationException /* unused ioe */)
{
// The process has already exited
// -or-
// There is no process associated with this Process object.
}
catch (Exception eInner)
{
//
// all other exceptions
//
DryadLogger.LogError(0, eInner, "Exception calling back to '{0}'", r.replyUri);
}
}
//
// If a property was handled from the graph manager, decrement the waiter count
//
if (ReplyDispatcher.IsGraphMrgUri(r.replyUri))
{
int n = Interlocked.Decrement(ref propertyWaiters);
DryadLogger.LogInformation("SetGetProp Thread", "Process {0} propertyWaiters = {1}", DryadId, n);
}
lock (syncRoot)
{
//
// If vertex process has exited, and sending vertex completed event, we can stop worrying
//
if (!finalStatusMessageSent)
{
if (latestVertexStatusSent != null)
{
if (!String.IsNullOrEmpty(latestVertexStatusSent.propertyString))
{
if (latestVertexStatusSent.propertyString.Contains(string.Format(@"(0x{0:x8})", Constants.DrError_VertexCompleted)))
{
finalStatusMessageSent = true;
}
}
}
}
}
DryadLogger.LogMethodExit();
}
/// <summary>
/// Asynchronously called on start command
/// </summary>
/// <param name="obj"></param>
void StartProcessThreadProc(Object obj)
{
ManualResetEvent serviceInitializedEvent = obj as ManualResetEvent;
bool started = false;
try
{
//
// Wait for service initialization
//
serviceInitializedEvent.WaitOne();
if (ExecutionHelper.InitializeForProcessExecution(dryadProcessId, Environment.GetEnvironmentVariable("XC_RESOURCEFILES")))
{
//
// Vertex working directory configured successfully, start the vertex host
//
environment.Add(Constants.vertexSvcLocalAddrEnvVar, localAddress);
ProcessStartInfo startInfo = new ProcessStartInfo();
startInfo.CreateNoWindow = true;
startInfo.UseShellExecute = false;
startInfo.WorkingDirectory = ProcessPathHelper.ProcessPath(dryadProcessId);
//YARN Debugging
//var procEnvVarKeys = startInfo.EnvironmentVariables.Keys;
//foreach (string key in procEnvVarKeys)
//{
// DryadLogger.LogInformation("StartProcess", "key: '{0}' value: '{1}'", key, startInfo.EnvironmentVariables[key]);
//}
string[] args = commandLine.Split(' ');
string arg = "";
for (int i = 1; i < args.Length; i++)
{
arg += args[i] + " ";
}
//
// Use either FQ path or path relative to job path
//
if (Path.IsPathRooted(args[0]))
{
startInfo.FileName = args[0];
}
else
{
startInfo.FileName = Path.Combine(ProcessPathHelper.JobPath, args[0]);
}
DryadLogger.LogInformation("StartProcess", "FileName: '{0}'", startInfo.FileName);
//
// Add environment variable to vertex host process
//
startInfo.Arguments = arg;
foreach (DictionaryEntry entry in environment)
{
string key = entry.Key.ToString();
if (key == null || startInfo.EnvironmentVariables.ContainsKey(key))
{
DryadLogger.LogInformation("StartProcess", "Attempting to add existing key '{0}' with value '{1}'",
entry.Key, entry.Value);
}
else
{
startInfo.EnvironmentVariables.Add(key, entry.Value.ToString());
}
}
lock (syncRoot)
{
//
// After taking lock, start the vertex host process and set up exited event handler
//
if (cancelled)
{
// If we've already been canceled, don't start the process
DryadLogger.LogInformation("Process start", "Not starting process {0} due to receipt of cancellation", DryadId);
return;
}
else
{
systemProcess = new Process();
systemProcess.StartInfo = startInfo;
systemProcess.EnableRaisingEvents = true;
systemProcess.Exited += new EventHandler(Process_Exited);
Console.WriteLine("Process start - Vertex host process starting");
started = systemProcess.Start();
Console.WriteLine("Process start - Vertex host process started");
if (started)
{
DryadLogger.LogInformation("Process start", "Vertex host process started");
state = ProcessState.Running;
}
else
{
DryadLogger.LogError(0, null, "Vertex host process failed to start");
}
}
}
}
else
{
DryadLogger.LogError(0, null, "Initialization failed");
}
}
catch (Exception e)
{
DryadLogger.LogError(0, e, "Error starting vertex");
}
if (started)
{
//
// Notify Graph Manager that process started if successful
//
bool success = ReplyDispatcher.FireStateChange(this.graphManagerReplyUri, this.dryadProcessId, ProcessState.Running);
if (!success)
{
//
// Graph manager doesn't know we started and we have no way to tell it, so it's
// best to just fail the vertex service task and let the job manager inform the graph manager
//
VertexService.Surrender(new Exception("Unable to communicate with graph manager."));
}
}
else
{
//
// Otherwise, notify GM that process has failed
//
lock (syncRoot)
{
// If we've already been canceled, we don't need to change state or record the initialization failure
if (!cancelled)
{
state = ProcessState.Completed;
this.failed = true;
exitCode = unchecked ((int)Constants.DrError_VertexInitialization); // DryadError_VertexInitialization
}
}
if (failed) // This also means we weren't canceled
{
// Notify the Graph Manager that the process failed to start
Process_Exited(this, null);
}
}
//
// Make sure process start event is set
//
processStartEvent.Set();
}