/// <summary>
/// Constructor - called when service first hosted
/// </summary>
public VertexService()
{
DryadLogger.LogMethodEntry();
this.vertexProcessTable = new SynchronizedCollection <VertexProcess>();
System.Threading.ThreadPool.QueueUserWorkItem(new WaitCallback(InitializationThreadProc));
DryadLogger.LogMethodExit();
}
/// <summary>
/// Cancels the vertex process with the provided id
/// </summary>
/// <param name="processId">vertex process id</param>
void IDryadVertexService.CancelScheduleProcess(int processId)
{
VertexProcess vp = null;
DryadLogger.LogMethodEntry(processId);
try
{
vp = FindByDryadId(processId);
if (vp != null)
{
vp.Cancel(false);
}
else
{
DryadLogger.LogWarning("Cancel Process", "Unknown process id {0}", processId);
}
}
catch (Exception e)
{
DryadLogger.LogWarning("Cancel Process", "Operation threw exception: {0}", e.ToString());
}
DryadLogger.LogMethodExit();
}
/// <summary>
/// Update properties
/// </summary>
/// <param name="replyEpr">callback URI</param>
/// <param name="processId">vertex process id</param>
/// <param name="infos">property information</param>
/// <param name="blockOnLabel">property update label</param>
/// <param name="blockOnVersion">property update version</param>
/// <param name="maxBlockTime">maximum time to wait for update</param>
/// <param name="getPropLabel">property to get</param>
/// <param name="ProcessStatistics">vertex host process statistics</param>
/// <returns>success/failure of property update</returns>
bool IDryadVertexService.SetGetProps(string replyEpr, int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics)
{
DryadLogger.LogMethodEntry(replyEpr, processId);
bool success = false;
try
{
// Get the vertex process ID
VertexProcess vp = FindByDryadId(processId);
if (vp != null)
{
success = vp.SetGetProps(replyEpr, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics);
}
else
{
DryadLogger.LogError(0, null, "Failed to set / get process properties: Unknown process id {0}", processId);
}
}
catch (Exception e)
{
DryadLogger.LogWarning("Set Or Get Process Properties", "Operation threw exception: {0}", e.ToString());
throw new FaultException <VertexServiceError>(new VertexServiceError("SetGetProps", e.ToString()));
}
DryadLogger.LogMethodExit(success);
return(success);
}
void ISchedulerHelper.StopTaskMonitorThread()
{
DryadLogger.LogMethodEntry();
bool wait = false;
if (m_taskMonitorThreadRunning)
{
lock (m_lock)
{
if (m_taskMonitorThreadRunning)
{
m_threadStopEvt.Set();
wait = true;
}
}
}
m_appMaster.Finish(true);
if (wait)
{
try
{
m_taskMonitorThread.Join();
}
catch (Exception e)
{
DryadLogger.LogError(0, e, "Failed to wait for task monitor thread to stop.");
}
}
DryadLogger.LogMethodExit();
}
/// <summary>
/// Removes reference to a vertex process
/// </summary>
/// <param name="processId">process id to forget</param>
void IDryadVertexService.ReleaseProcess(int processId)
{
DryadLogger.LogMethodEntry(processId);
VertexProcess vp = null;
try
{
vp = FindByDryadId(processId);
if (vp != null)
{
vertexProcessTable.Remove(vp);
vp.Dispose();
}
else
{
DryadLogger.LogWarning("Release Process", "Unknown process id {0}", processId);
}
}
catch (Exception e)
{
DryadLogger.LogWarning("Release Process", "Operation threw exception: {0}", e.ToString());
}
DryadLogger.LogMethodExit();
}
//
// Asynchronously start vertex process
//
public bool Start(ManualResetEvent serviceInitializedEvent)
{
DryadLogger.LogMethodEntry(this.DryadId);
bool result = ThreadPool.QueueUserWorkItem(new WaitCallback(StartProcessThreadProc), serviceInitializedEvent);
DryadLogger.LogMethodExit(result);
return(result);
}
/// <summary>
/// Fail the vertex service task
/// </summary>
internal static void Surrender(Exception ex)
{
DryadLogger.LogMethodEntry();
ReplyDispatcher.ShuttingDown = true;
VertexService.internalShutdown = true;
VertexService.ShutdownReason = ex;
VertexService.shutdownEvent.Set();
DryadLogger.LogMethodExit();
}
/// <summary>
///
/// </summary>
public void Dispose()
{
DryadLogger.LogMethodEntry(this.DryadId);
Dispose(true);
GC.SuppressFinalize(this);
DryadLogger.LogMethodExit();
}
/// <summary>
/// Attempt to call SetGetPropsComplete on specified WCF service.
/// </summary>
/// <param name="replyUri">Service endpoint</param>
/// <param name="systemProcess"></param>
/// <param name="processId"></param>
/// <param name="info"></param>
/// <param name="propertyLabels"></param>
/// <param name="propertyVersions"></param>
/// <returns></returns>
public static bool SetGetPropsComplete(string replyUri, Process systemProcess, int processId, ProcessInfo info, string[] propertyLabels, ulong[] propertyVersions)
{
DryadLogger.LogMethodEntry(replyUri, processId);
bool result = false;
VertexCallbackServiceClient client = GetClient(replyUri);
//
// Try to set/get properties up to numRetries times
//
for (int index = 0; index < numRetries; index++)
{
try
{
//
// If client is null, try reopening it
//
if (client == null)
{
client = CreateClient(replyUri);
}
//
// Make SetGetPropsComplete WCF call, return success
//
client.SetGetPropsComplete(processId, info, propertyLabels, propertyVersions);
result = true;
break;
}
catch (Exception e)
{
if ((IsGraphMrgUri(replyUri) == false && systemProcess.HasExited) || shuttingDown)
{
//
// If trying to connect to non-running vertex or job is shutting down, don't retry and report success.
//
DisposeClient(ref client);
return(true);
}
else
{
//
// If call failed and talking to GM or running vertex process, try reopening WCF client and calling again
//
client = ReopenClientForRetry(replyUri, e);
}
}
}
//
// If failed to connect X times, report error
//
DryadLogger.LogMethodExit(result);
return(result);
}
/// <summary>
/// Set process state to cancelled and stop the vertex host process if possible
/// </summary>
public void Cancel(bool suppressNotifications)
{
DryadLogger.LogMethodEntry(this.DryadId);
lock (syncRoot)
{
if (state == ProcessState.Completed)
{
// Process has already completed before cancelation made it here, do nothing
DryadLogger.LogInformation("Cancel process", "Process {0} has already exited", DryadId);
DryadLogger.LogMethodExit();
return;
}
DryadLogger.LogInformation("Cancel process", "Process {0} has not already exited", DryadId);
state = ProcessState.Completed;
this.cancelled = true;
}
// If the process started, kill it
if (systemProcess != null)
{
try
{
// Killing the process will trigger Process_Exited
DryadLogger.LogInformation("Cancel process", "Killing system process for process id {0}", DryadId);
if (suppressNotifications)
{
// Remove the Exited event handler
systemProcess.Exited -= this.Process_Exited;
}
systemProcess.Kill();
DryadLogger.LogMethodExit();
return;
}
catch (Exception e)
{
//
// Failed to kill process - log exception
//
DryadLogger.LogError(0, e, "Failed to kill system process for process id {0}", DryadId);
}
}
else
{
DryadLogger.LogInformation("Cancel process", "Process {0} has not started yet", DryadId);
}
// Process was either not running or failed to die, trigger Process_Exited ourself
if (!suppressNotifications)
{
Process_Exited(this, null);
}
DryadLogger.LogMethodExit();
}
/// <summary>
/// Schedule a vertex host process using the provided parameters
/// </summary>
/// <param name="replyUri">callback URI</param>
/// <param name="processId">vertex process id</param>
/// <param name="commandLine">vertex host command line</param>
/// <param name="environment">vertex host environment variables</param>
/// <returns>Success/Failure of starting vertex process thread</returns>
bool IDryadVertexService.ScheduleProcess(string replyUri, int processId, string commandLine, StringDictionary environment)
{
DryadLogger.LogMethodEntry(processId, commandLine);
bool startSuccess = false;
Console.WriteLine("Starting process id {0} with commandLIne: '{1}", processId, commandLine);
try
{
VertexProcess newProcess = null;
lock (vertexProcessTable.SyncRoot)
{
foreach (VertexProcess vp in vertexProcessTable)
{
if (vp.DryadId == processId)
{
// This means a previous call to Schedule process partially succeeded:
// the call made it to the service but something went wrong with the response
// so the GM's xcompute machinery retried the call. We can just return success
// for this case rather than tearing down the process and creating a new one.
return(true);
}
if (vp.State <= ProcessState.Running)
{
// There should be no other processes running.
// If there are, it means a previous communication error
// cause the GM to give up on this node for a while.
// Kill anything that's still hanging around.
vp.Cancel(true);
}
}
newProcess = new VertexProcess(
replyUri,
processId,
commandLine,
environment,
OperationContext.Current.Channel.LocalAddress.Uri.ToString()
);
this.vertexProcessTable.Add(newProcess);
}
startSuccess = newProcess.Start(initializedEvent);
}
catch (Exception e)
{
DryadLogger.LogWarning("Schedule Process", "Operation threw exception: {0}", e.ToString());
throw new FaultException <VertexServiceError>(new VertexServiceError("ReleaseProcess", e.ToString()));
}
DryadLogger.LogMethodExit(startSuccess);
return(startSuccess);
}
/// <summary>
/// Notify GM that vertex host process exited
/// </summary>
/// <param name="replyUri">GM address</param>
/// <param name="processId">vertex process id</param>
/// <param name="exitCode">reason for vertex host exit</param>
/// <returns>success/failure</returns>
public static bool ProcessExited(string replyUri, int processId, int exitCode)
{
DryadLogger.LogMethodEntry(replyUri, processId, exitCode);
bool result = false;
VertexCallbackServiceClient client = GetClient(replyUri);
//
// Try to notify GM that the process has exited up to numRetries times
//
for (int index = 0; index < numRetries; index++)
{
try
{
//
// If client is null, try reopening it
//
if (client == null)
{
client = CreateClient(replyUri);
}
//
// Make ProcessExited WCF call, return success
//
client.ProcessExited(processId, exitCode);
result = true;
break;
}
catch (Exception e)
{
if (shuttingDown)
{
// if shutting down, just return
DisposeClient(ref client);
return(true);
}
else
{
//
// If call failed, try reopening WCF client and calling again
//
client = ReopenClientForRetry(replyUri, e);
}
}
}
//
// If failure occurs after X retry attempts, report error
//
DryadLogger.LogMethodExit(result);
return(result);
}
/// <summary>
/// Vertex host process exited event - marks process state and queues up exit process thread
/// </summary>
/// <param name="sender"></param>
/// <param name="args"></param>
private void Process_Exited(object sender, EventArgs args)
{
DryadLogger.LogMethodEntry(DryadId);
// Ensure the process exited code can only be executed once
lock (syncRoot)
{
if (exited)
{
DryadLogger.LogInformation("Process exit", "Process {0} already exited", DryadId);
DryadLogger.LogMethodExit();
return;
}
exited = true;
}
if (cancelled)
{
DryadLogger.LogInformation("Process exit", "Process {0} was cancelled", DryadId);
exitCode = unchecked ((int)0x830A0003); // DrError_VertexReceivedTermination
}
else
{
exitCode = systemProcess.ExitCode;
DryadLogger.LogInformation("Process exit", "Process {0} exit code {1}", DryadId, exitCode);
if (exitCode == 0)
{
lock (syncRoot)
{
state = ProcessState.Completed;
}
}
else
{
lock (syncRoot)
{
state = ProcessState.Completed;
this.failed = true;
}
}
}
//
// Ensure that the vertex complete event is sent to GM and that all pending properties are handled
//
ThreadPool.QueueUserWorkItem(new WaitCallback(ExitProcessThreadProc));
DryadLogger.LogMethodExit();
}
/// <summary>
/// Initialize the endpoint addresses for each vertex host
/// </summary>
/// <param name="vertexEndpointAddresses">List of vertex host addresses</param>
void IDryadVertexService.Initialize(StringDictionary vertexEndpointAddresses)
{
DryadLogger.LogMethodEntry(vertexEndpointAddresses.Count);
try
{
this.vertexEndpointAddresses = vertexEndpointAddresses;
}
catch (Exception e)
{
DryadLogger.LogWarning("Initialize", "Operation threw exception: {0}", e.ToString());
}
DryadLogger.LogMethodExit();
}
/// <summary>
/// Shut down the vertex service
/// </summary>
/// <param name="ShutdownCode"></param>
void IDryadVertexService.Shutdown(uint ShutdownCode)
{
DryadLogger.LogMethodEntry(ShutdownCode);
try
{
ReplyDispatcher.ShuttingDown = true;
VertexService.shutdownEvent.Set();
}
catch (Exception e)
{
DryadLogger.LogWarning("Shutdown", "Operation threw exception: {0}", e.ToString());
}
DryadLogger.LogMethodExit();
}
/// <summary>
/// Make sure all pending properties get handled and vertex complete event is sent to GM
/// </summary>
/// <param name="obj"></param>
void ExitProcessThreadProc(Object obj)
{
DryadLogger.LogMethodEntry();
//
// Wait until all property waiters have been notified and the final
// status message sent, iff the process completed successfully
//
do
{
//
// Clear any thing intended for the vertex
//
SetAllPropertyWaiters();
lock (syncRoot)
{
// If nobody is waiting, AND
if (propertyWaiters == 0)
{
// Process did not complete successfully OR
// final status message has already been sent
if (!Succeeded || finalStatusMessageSent)
{
// Then we can send the Process Exit notification
break;
}
}
}
Thread.Sleep(10);
} while(true);
ReplyDispatcher.ProcessExited(this.graphManagerReplyUri, this.dryadProcessId, this.exitCode);
//
// This should never happen unless a property is requested after the vertex completed event is sent
// so it's not a big deal if it does because the GM knows that the vertex is done
//
if (propertyWaiters > 0)
{
DryadLogger.LogWarning("Process exit", "Leaving thread with {0} property waiter(s).", propertyWaiters);
}
DryadLogger.LogMethodExit();
}
/// <summary>
///
/// </summary>
/// <param name="disposing"></param>
private void Dispose(bool disposing)
{
DryadLogger.LogMethodEntry(disposing);
if (!this.m_disposed)
{
lock (syncRoot)
{
if (!this.m_disposed)
{
if (disposing)
{
// Close start event handle
try
{
processStartEvent.Close();
}
catch (Exception ex)
{
DryadLogger.LogError(0, ex);
}
// Close any get/set property wait handles
foreach (KeyValuePair <string, Dictionary <ulong, ManualResetEvent> > label in propertyWaitEvents)
{
foreach (KeyValuePair <ulong, ManualResetEvent> version in label.Value)
{
try
{
version.Value.Close();
}
catch (Exception ex)
{
DryadLogger.LogError(0, ex);
}
}
}
propertyWaitEvents.Clear();
}
m_disposed = true;
}
}
}
DryadLogger.LogMethodExit();
}
/// <summary>
/// Creates a request for property update from one vertex to another (one vertex is usually GM)
/// </summary>
/// <param name="replyEpr">Address to send response to</param>
/// <param name="infos"></param>
/// <param name="blockOnLabel"></param>
/// <param name="blockOnVersion"></param>
/// <param name="maxBlockTime"></param>
/// <param name="getPropLabel"></param>
/// <param name="ProcessStatistics"></param>
/// <returns>Returns success/failure of thread startup</returns>
public bool SetGetProps(string replyEpr, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics)
{
DryadLogger.LogMethodEntry(replyEpr, this.DryadId);
//
// Check if graph manager is the one calling. If so, increment propertyWaiter count
//
if (ReplyDispatcher.IsGraphMrgUri(replyEpr))
{
int n = Interlocked.Increment(ref propertyWaiters);
}
//
// Create new property request with provided parameters and queue up request sending
//
PropertyRequest req = new PropertyRequest(replyEpr, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics);
bool result = ThreadPool.QueueUserWorkItem(new WaitCallback(SetGetPropThreadProc), req);
DryadLogger.LogMethodExit(result);
return(result);
}
public bool Start(string listenUri, ISchedulerHelper schedulerHelper)
{
DryadLogger.LogMethodEntry(listenUri);
Uri baseAddress = new Uri(listenUri);
try
{
NetTcpBinding binding = schedulerHelper.GetVertexServiceBinding();
selfHost = null;
// Retry opening the service port if address is already in use
int maxRetryCount = 20; // Results in retrying for ~1 min
for (int retryCount = 0; retryCount < maxRetryCount; retryCount++)
{
try
{
//Step 1 of the hosting procedure: Create ServiceHost
selfHost = new ServiceHost(callbackService, baseAddress);
//Step 2 of the hosting procedure: Add service endpoints.
ServiceEndpoint vertexEndpoint = selfHost.AddServiceEndpoint(typeof(IDryadVertexCallback), binding, Constants.vertexCallbackServiceName);
ServiceThrottlingBehavior stb = new ServiceThrottlingBehavior();
stb.MaxConcurrentCalls = Constants.MaxConnections;
stb.MaxConcurrentSessions = Constants.MaxConnections;
selfHost.Description.Behaviors.Add(stb);
//Step 3 of hosting procedure : Add a security manager
selfHost.Authorization.ServiceAuthorizationManager = new DryadVertexServiceAuthorizationManager();
// Step 4 of the hosting procedure: Start the service.
selfHost.Open();
break;
}
catch (AddressAlreadyInUseException)
{
if (selfHost != null)
{
selfHost.Abort();
selfHost = null;
}
// If this is the last try, dont sleep. Just rethrow exception to exit.
if (retryCount < maxRetryCount - 1)
{
DryadLogger.LogInformation("Start Vertex Callback Service", "Address already in use. Retrying...");
System.Threading.Thread.Sleep(3000);
}
else
{
throw;
}
}
}
DryadLogger.LogInformation("Start Vertex Callback Service", "Service Host started successfully");
return(true);
}
catch (CommunicationException ce)
{
DryadLogger.LogCritical(0, ce, "Failed to start vertex callback service");
try
{
if (selfHost != null)
{
selfHost.Abort();
}
}
catch
{
}
return(false);
}
}
/// <summary>
/// Adds specified property to property wait list and waits for it.
/// </summary>
/// <param name="blockOnLabel">Property label to wait for</param>
/// <param name="blockOnVersion">Version of property to wait for</param>
/// <param name="maxBlockTime">Time to wait for property</param>
/// <returns>False if property was requested but none was returned</returns>
private bool BlockOnProperty(string blockOnLabel, ulong blockOnVersion, long maxBlockTime)
{
DryadLogger.LogMethodEntry();
//
// Return true if no label is provided
//
if (String.IsNullOrEmpty(blockOnLabel))
{
DryadLogger.LogMethodExit(true);
return(true);
}
DryadLogger.LogInformation("Block on property", "Label {0} Version {1} maxBlockTime {2}", blockOnLabel, blockOnVersion, maxBlockTime);
ProcessPropertyInfo prop = null;
//
// If the process already exited, don't bother adding a wait event for
// this property - if it's not already set it never will be.
//
lock (syncRoot)
{
if (!exited)
{
//
// Add this label and version to the wait events list if needed
//
if (propertyWaitEvents.ContainsKey(blockOnLabel) == false)
{
propertyWaitEvents.Add(blockOnLabel, new Dictionary <ulong, ManualResetEvent>());
}
if (propertyWaitEvents[blockOnLabel].ContainsKey(blockOnVersion) == false)
{
propertyWaitEvents[blockOnLabel].Add(blockOnVersion, new ManualResetEvent(false));
}
}
else
{
DryadLogger.LogInformation("Block on property", "Process {0} already exited, not adding waiter", this.DryadId);
}
}
// todo: We still may want to implement timeouts to deal with deadlocks in the service / host but it hasn't been an issue yet.
//if (propertyWaitEvents[blockOnLabel][blockOnVersion].WaitOne(new TimeSpan(maxBlockTime), false))
//
// Wait forever (or until process exits or is disposed) for the property to be set or interrupted
//
while (!exited)
{
try
{
if (propertyWaitEvents[blockOnLabel][blockOnVersion].WaitOne(100, false))
{
break;
}
}
catch (ObjectDisposedException)
{
DryadLogger.LogWarning("Block on property", "Process {0} disposed while waiting for label {1}, version {2}", DryadId, blockOnLabel, blockOnVersion);
DryadLogger.LogMethodExit(false);
return(false);
}
}
// Did we get the property, or did the process
// terminate?
int index;
if (TryGetProperty(blockOnLabel, out prop, out index))
{
//
// If a property was successfully returned, return true
//
if ((blockOnVersion == 0) || (prop.propertyVersion > blockOnVersion))
{
DryadLogger.LogMethodExit(true);
return(true);
}
if (state == ProcessState.Completed)
{
DryadLogger.LogInformation("Block on property", "Vertex completed (wait) requested version:{0} returned version:{1} of label {2}", blockOnVersion, prop.propertyVersion, blockOnLabel);
DryadLogger.LogMethodExit(true);
return(true);
}
}
//
// Return false if property was requested but none was found
//
DryadLogger.LogMethodExit(false);
return(false);
}
/// <summary>
/// Set a property and record the version
/// </summary>
/// <param name="property"></param>
/// <param name="newVersion"></param>
private void SetProperty(ProcessPropertyInfo property, out ulong newVersion)
{
DryadLogger.LogMethodEntry(property.propertyLabel);
ProcessPropertyInfo oldProperty = null;
lock (syncRoot)
{
int index;
if (TryGetProperty(property.propertyLabel, out oldProperty, out index))
{
//
// If property found in local array, then we are setting a new version of existing property
// Copy the new property information into the array
//
oldProperty.propertyVersion++;
newVersion = oldProperty.propertyVersion;
if (property.propertyBlock != null && property.propertyBlock.Length > 0)
{
oldProperty.propertyBlock = property.propertyBlock;
}
oldProperty.propertyString = property.propertyString;
CopyProp(oldProperty, out info.propertyInfos[index]);
}
else
{
//
// if property not found in local array, then setting a new property
// use version 1, unless valid value specified
//
if (property.propertyVersion == ulong.MaxValue || property.propertyVersion == 0)
{
property.propertyVersion = 1;
}
newVersion = property.propertyVersion;
//
// Create or resize the local info array as necessary and append this property
//
if (info.propertyInfos == null)
{
info.propertyInfos = new ProcessPropertyInfo[1];
}
else
{
Array.Resize(ref info.propertyInfos, info.propertyInfos.Length + 1);
}
info.propertyInfos[info.propertyInfos.Length - 1] = property;
}
//
// If there was a vertex completed event, record the latest vertex status
//
if (StatusMessageContainsDryadError_VertexCompleted(property.propertyLabel))
{
CopyProp(property, out latestVertexStatusReceived);
latestVertexStatusReceived.propertyVersion = newVersion;
}
//
// Wake up anyone waiting for a property change by adding a new wait event for this property if needed
//
if (propertyWaitEvents.ContainsKey(property.propertyLabel) == false)
{
propertyWaitEvents.Add(property.propertyLabel, new Dictionary <ulong, ManualResetEvent>());
}
//
// Wake up anyone waiting for this version of the property
//
if (propertyWaitEvents[property.propertyLabel].ContainsKey(newVersion - 1))
{
propertyWaitEvents[property.propertyLabel][newVersion - 1].Set();
}
else
{
propertyWaitEvents[property.propertyLabel].Add(newVersion - 1, new ManualResetEvent(true));
}
//
// Wake up anyone waiting for any version of this property
//
if (newVersion > 1)
{
if (propertyWaitEvents[property.propertyLabel].ContainsKey(0))
{
propertyWaitEvents[property.propertyLabel][0].Set();
}
}
}
DryadLogger.LogMethodExit();
}
/// <summary>
/// Called in new thread in setgetproperty service operation
/// </summary>
/// <param name="obj"></param>
void SetGetPropThreadProc(Object obj)
{
DryadLogger.LogMethodEntry(DryadId);
PropertyRequest r = obj as PropertyRequest;
ProcessInfo infoLocal = new ProcessInfo();
ulong[] propertyVersions = null;
string[] propertyLabels = null;
//
// Make sure process is started before continuing
//
if (this.State < ProcessState.Running)
{
try
{
processStartEvent.WaitOne();
}
catch (ObjectDisposedException ex)
{
// The process was cancelled and released before it started running, just return
if (exited)
{
DryadLogger.LogInformation("SetGetProp Thread", "Process {0} cancelled or exited before starting.", this.DryadId);
}
else
{
DryadLogger.LogError(0, ex);
}
DryadLogger.LogMethodExit();
return;
}
}
//
// Use status_pending if running, vertex initialization failure if process is failed and process exit code otherwise
//
infoLocal.processStatus = 0x103; // WinNT.h STATUS_PENDING
infoLocal.processState = state;
if (state == ProcessState.Running)
{
infoLocal.exitCode = 0x103; // WinNT.h STATUS_PENDING
}
else if (failed)
{
infoLocal.exitCode = Constants.DrError_VertexError;
}
else if (cancelled)
{
infoLocal.exitCode = Constants.DrError_VertexReceivedTermination; // DryadError_VertexReceivedTermination
}
else
{
infoLocal.exitCode = (uint)systemProcess.ExitCode;
}
//
// Record specified properties and update versions - wakes up anyone waiting for property changes
//
SetProperties(r.infos, out propertyLabels, out propertyVersions);
//
// Try to get property update
//
if (BlockOnProperty(r.blockOnLabel, r.blockOnVersion, r.maxBlockTime))
{
//
// If property update was received, update the received property information
// If received property marks vertex completed, record that
//
if (r.getPropLabel != null && r.getPropLabel.Length > 0)
{
lock (syncRoot)
{
infoLocal.propertyInfos = new ProcessPropertyInfo[1];
int index;
if (TryGetProperty(r.getPropLabel, out infoLocal.propertyInfos[0], out index) == false)
{
DryadLogger.LogError(0, null, "Failed to get property for label {0}", r.getPropLabel);
}
if (StatusMessageContainsDryadError_VertexCompleted(infoLocal.propertyInfos[0].propertyLabel))
{
CopyProp(infoLocal.propertyInfos[0], out latestVertexStatusSent);
}
}
}
//
// If request asks for statistics on vertex process, get them
//
if (r.ProcessStatistics)
{
if (GetStatistics(out infoLocal.processStatistics) == false)
{
DryadLogger.LogError(0, null, "Failed to get vertex statistics");
}
}
}
//
// Try to report property change, if unsuccessful, kill the running vertex host process
//
if (!ReplyDispatcher.SetGetPropsComplete(r.replyUri, systemProcess, dryadProcessId, infoLocal, propertyLabels, propertyVersions))
{
try
{
systemProcess.Kill();
}
catch (InvalidOperationException /* unused ioe */)
{
// The process has already exited
// -or-
// There is no process associated with this Process object.
}
catch (Exception eInner)
{
//
// all other exceptions
//
DryadLogger.LogError(0, eInner, "Exception calling back to '{0}'", r.replyUri);
}
}
//
// If a property was handled from the graph manager, decrement the waiter count
//
if (ReplyDispatcher.IsGraphMrgUri(r.replyUri))
{
int n = Interlocked.Decrement(ref propertyWaiters);
DryadLogger.LogInformation("SetGetProp Thread", "Process {0} propertyWaiters = {1}", DryadId, n);
}
lock (syncRoot)
{
//
// If vertex process has exited, and sending vertex completed event, we can stop worrying
//
if (!finalStatusMessageSent)
{
if (latestVertexStatusSent != null)
{
if (!String.IsNullOrEmpty(latestVertexStatusSent.propertyString))
{
if (latestVertexStatusSent.propertyString.Contains(string.Format(@"(0x{0:x8})", Constants.DrError_VertexCompleted)))
{
finalStatusMessageSent = true;
}
}
}
}
}
DryadLogger.LogMethodExit();
}
/// <summary>
/// Gets information about the vertex service
/// </summary>
/// <returns></returns>
VertexStatus IDryadVertexService.CheckStatus()
{
DryadLogger.LogMethodEntry();
VertexStatus status = new VertexStatus();
status.serviceIsAlive = true;
//
// Update information about disk usage
//
foreach (string disk in Environment.GetLogicalDrives())
{
ulong freeDiskSpaceforUser;
ulong totalDiskSpace;
ulong freeDiskSpace;
if (NativeMethods.GetDiskFreeSpaceEx(disk, out freeDiskSpaceforUser, out totalDiskSpace, out freeDiskSpace))
{
status.freeDiskSpaces.Add(disk, freeDiskSpace);
}
else
{
//
// Report any errors as warnings, as this is a non-essential call
//
int errorCode = Marshal.GetLastWin32Error();
Exception lastex = Marshal.GetExceptionForHR(errorCode);
if (lastex != null)
{
DryadLogger.LogWarning("Unable to get disk space information", "Disk: {0} Error: {1}", disk, lastex.Message);
}
else
{
DryadLogger.LogWarning("Unable to get disk space information", "Disk: {0} Error Code: {1}", disk, errorCode);
}
}
}
//
// Update information about memory usage
//
NativeMethods.MEMORYSTATUSEX memStatus = new NativeMethods.MEMORYSTATUSEX();
if (NativeMethods.GlobalMemoryStatusEx(memStatus))
{
status.freePhysicalMemory = memStatus.ullAvailPhys;
status.freeVirtualMemory = memStatus.ullAvailVirtual;
}
else
{
//
// Report any errors as warnings, as this is a non-essential call
//
int errorCode = Marshal.GetLastWin32Error();
Exception lastex = Marshal.GetExceptionForHR(errorCode);
if (lastex != null)
{
DryadLogger.LogWarning("Unable to get memory information", "Error: {0}", lastex.Message);
}
else
{
DryadLogger.LogWarning("Unable to get memory information", "Error Code: {0}", errorCode);
}
}
//
// Get process info for each running vertex process
//
status.runningProcessCount = 0;
lock (vertexProcessTable.SyncRoot)
{
foreach (VertexProcess vp in this.vertexProcessTable)
{
VertexProcessInfo vpInfo = new VertexProcessInfo();
vpInfo.DryadId = vp.DryadId;
vpInfo.commandLine = vp.commandLine;
vpInfo.State = vp.State;
status.vps.Add(vpInfo);
if (vp.State == ProcessState.Running)
{
status.runningProcessCount++;
}
}
}
DryadLogger.LogMethodExit(status);
return(status);
}
