/// <summary> /// Constructor - called when service first hosted /// </summary> public VertexService() { DryadLogger.LogMethodEntry(); this.vertexProcessTable = new SynchronizedCollection <VertexProcess>(); System.Threading.ThreadPool.QueueUserWorkItem(new WaitCallback(InitializationThreadProc)); DryadLogger.LogMethodExit(); }
/// <summary> /// Cancels the vertex process with the provided id /// </summary> /// <param name="processId">vertex process id</param> void IDryadVertexService.CancelScheduleProcess(int processId) { VertexProcess vp = null; DryadLogger.LogMethodEntry(processId); try { vp = FindByDryadId(processId); if (vp != null) { vp.Cancel(false); } else { DryadLogger.LogWarning("Cancel Process", "Unknown process id {0}", processId); } } catch (Exception e) { DryadLogger.LogWarning("Cancel Process", "Operation threw exception: {0}", e.ToString()); } DryadLogger.LogMethodExit(); }
/// <summary> /// Update properties /// </summary> /// <param name="replyEpr">callback URI</param> /// <param name="processId">vertex process id</param> /// <param name="infos">property information</param> /// <param name="blockOnLabel">property update label</param> /// <param name="blockOnVersion">property update version</param> /// <param name="maxBlockTime">maximum time to wait for update</param> /// <param name="getPropLabel">property to get</param> /// <param name="ProcessStatistics">vertex host process statistics</param> /// <returns>success/failure of property update</returns> bool IDryadVertexService.SetGetProps(string replyEpr, int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics) { DryadLogger.LogMethodEntry(replyEpr, processId); bool success = false; try { // Get the vertex process ID VertexProcess vp = FindByDryadId(processId); if (vp != null) { success = vp.SetGetProps(replyEpr, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics); } else { DryadLogger.LogError(0, null, "Failed to set / get process properties: Unknown process id {0}", processId); } } catch (Exception e) { DryadLogger.LogWarning("Set Or Get Process Properties", "Operation threw exception: {0}", e.ToString()); throw new FaultException <VertexServiceError>(new VertexServiceError("SetGetProps", e.ToString())); } DryadLogger.LogMethodExit(success); return(success); }
void ISchedulerHelper.StopTaskMonitorThread() { DryadLogger.LogMethodEntry(); bool wait = false; if (m_taskMonitorThreadRunning) { lock (m_lock) { if (m_taskMonitorThreadRunning) { m_threadStopEvt.Set(); wait = true; } } } m_appMaster.Finish(true); if (wait) { try { m_taskMonitorThread.Join(); } catch (Exception e) { DryadLogger.LogError(0, e, "Failed to wait for task monitor thread to stop."); } } DryadLogger.LogMethodExit(); }
/// <summary> /// Removes reference to a vertex process /// </summary> /// <param name="processId">process id to forget</param> void IDryadVertexService.ReleaseProcess(int processId) { DryadLogger.LogMethodEntry(processId); VertexProcess vp = null; try { vp = FindByDryadId(processId); if (vp != null) { vertexProcessTable.Remove(vp); vp.Dispose(); } else { DryadLogger.LogWarning("Release Process", "Unknown process id {0}", processId); } } catch (Exception e) { DryadLogger.LogWarning("Release Process", "Operation threw exception: {0}", e.ToString()); } DryadLogger.LogMethodExit(); }
// // Asynchronously start vertex process // public bool Start(ManualResetEvent serviceInitializedEvent) { DryadLogger.LogMethodEntry(this.DryadId); bool result = ThreadPool.QueueUserWorkItem(new WaitCallback(StartProcessThreadProc), serviceInitializedEvent); DryadLogger.LogMethodExit(result); return(result); }
/// <summary> /// Fail the vertex service task /// </summary> internal static void Surrender(Exception ex) { DryadLogger.LogMethodEntry(); ReplyDispatcher.ShuttingDown = true; VertexService.internalShutdown = true; VertexService.ShutdownReason = ex; VertexService.shutdownEvent.Set(); DryadLogger.LogMethodExit(); }
/// <summary> /// /// </summary> public void Dispose() { DryadLogger.LogMethodEntry(this.DryadId); Dispose(true); GC.SuppressFinalize(this); DryadLogger.LogMethodExit(); }
/// <summary> /// Attempt to call SetGetPropsComplete on specified WCF service. /// </summary> /// <param name="replyUri">Service endpoint</param> /// <param name="systemProcess"></param> /// <param name="processId"></param> /// <param name="info"></param> /// <param name="propertyLabels"></param> /// <param name="propertyVersions"></param> /// <returns></returns> public static bool SetGetPropsComplete(string replyUri, Process systemProcess, int processId, ProcessInfo info, string[] propertyLabels, ulong[] propertyVersions) { DryadLogger.LogMethodEntry(replyUri, processId); bool result = false; VertexCallbackServiceClient client = GetClient(replyUri); // // Try to set/get properties up to numRetries times // for (int index = 0; index < numRetries; index++) { try { // // If client is null, try reopening it // if (client == null) { client = CreateClient(replyUri); } // // Make SetGetPropsComplete WCF call, return success // client.SetGetPropsComplete(processId, info, propertyLabels, propertyVersions); result = true; break; } catch (Exception e) { if ((IsGraphMrgUri(replyUri) == false && systemProcess.HasExited) || shuttingDown) { // // If trying to connect to non-running vertex or job is shutting down, don't retry and report success. // DisposeClient(ref client); return(true); } else { // // If call failed and talking to GM or running vertex process, try reopening WCF client and calling again // client = ReopenClientForRetry(replyUri, e); } } } // // If failed to connect X times, report error // DryadLogger.LogMethodExit(result); return(result); }
/// <summary> /// Set process state to cancelled and stop the vertex host process if possible /// </summary> public void Cancel(bool suppressNotifications) { DryadLogger.LogMethodEntry(this.DryadId); lock (syncRoot) { if (state == ProcessState.Completed) { // Process has already completed before cancelation made it here, do nothing DryadLogger.LogInformation("Cancel process", "Process {0} has already exited", DryadId); DryadLogger.LogMethodExit(); return; } DryadLogger.LogInformation("Cancel process", "Process {0} has not already exited", DryadId); state = ProcessState.Completed; this.cancelled = true; } // If the process started, kill it if (systemProcess != null) { try { // Killing the process will trigger Process_Exited DryadLogger.LogInformation("Cancel process", "Killing system process for process id {0}", DryadId); if (suppressNotifications) { // Remove the Exited event handler systemProcess.Exited -= this.Process_Exited; } systemProcess.Kill(); DryadLogger.LogMethodExit(); return; } catch (Exception e) { // // Failed to kill process - log exception // DryadLogger.LogError(0, e, "Failed to kill system process for process id {0}", DryadId); } } else { DryadLogger.LogInformation("Cancel process", "Process {0} has not started yet", DryadId); } // Process was either not running or failed to die, trigger Process_Exited ourself if (!suppressNotifications) { Process_Exited(this, null); } DryadLogger.LogMethodExit(); }
/// <summary> /// Schedule a vertex host process using the provided parameters /// </summary> /// <param name="replyUri">callback URI</param> /// <param name="processId">vertex process id</param> /// <param name="commandLine">vertex host command line</param> /// <param name="environment">vertex host environment variables</param> /// <returns>Success/Failure of starting vertex process thread</returns> bool IDryadVertexService.ScheduleProcess(string replyUri, int processId, string commandLine, StringDictionary environment) { DryadLogger.LogMethodEntry(processId, commandLine); bool startSuccess = false; Console.WriteLine("Starting process id {0} with commandLIne: '{1}", processId, commandLine); try { VertexProcess newProcess = null; lock (vertexProcessTable.SyncRoot) { foreach (VertexProcess vp in vertexProcessTable) { if (vp.DryadId == processId) { // This means a previous call to Schedule process partially succeeded: // the call made it to the service but something went wrong with the response // so the GM's xcompute machinery retried the call. We can just return success // for this case rather than tearing down the process and creating a new one. return(true); } if (vp.State <= ProcessState.Running) { // There should be no other processes running. // If there are, it means a previous communication error // cause the GM to give up on this node for a while. // Kill anything that's still hanging around. vp.Cancel(true); } } newProcess = new VertexProcess( replyUri, processId, commandLine, environment, OperationContext.Current.Channel.LocalAddress.Uri.ToString() ); this.vertexProcessTable.Add(newProcess); } startSuccess = newProcess.Start(initializedEvent); } catch (Exception e) { DryadLogger.LogWarning("Schedule Process", "Operation threw exception: {0}", e.ToString()); throw new FaultException <VertexServiceError>(new VertexServiceError("ReleaseProcess", e.ToString())); } DryadLogger.LogMethodExit(startSuccess); return(startSuccess); }
/// <summary> /// Notify GM that vertex host process exited /// </summary> /// <param name="replyUri">GM address</param> /// <param name="processId">vertex process id</param> /// <param name="exitCode">reason for vertex host exit</param> /// <returns>success/failure</returns> public static bool ProcessExited(string replyUri, int processId, int exitCode) { DryadLogger.LogMethodEntry(replyUri, processId, exitCode); bool result = false; VertexCallbackServiceClient client = GetClient(replyUri); // // Try to notify GM that the process has exited up to numRetries times // for (int index = 0; index < numRetries; index++) { try { // // If client is null, try reopening it // if (client == null) { client = CreateClient(replyUri); } // // Make ProcessExited WCF call, return success // client.ProcessExited(processId, exitCode); result = true; break; } catch (Exception e) { if (shuttingDown) { // if shutting down, just return DisposeClient(ref client); return(true); } else { // // If call failed, try reopening WCF client and calling again // client = ReopenClientForRetry(replyUri, e); } } } // // If failure occurs after X retry attempts, report error // DryadLogger.LogMethodExit(result); return(result); }
/// <summary> /// Vertex host process exited event - marks process state and queues up exit process thread /// </summary> /// <param name="sender"></param> /// <param name="args"></param> private void Process_Exited(object sender, EventArgs args) { DryadLogger.LogMethodEntry(DryadId); // Ensure the process exited code can only be executed once lock (syncRoot) { if (exited) { DryadLogger.LogInformation("Process exit", "Process {0} already exited", DryadId); DryadLogger.LogMethodExit(); return; } exited = true; } if (cancelled) { DryadLogger.LogInformation("Process exit", "Process {0} was cancelled", DryadId); exitCode = unchecked ((int)0x830A0003); // DrError_VertexReceivedTermination } else { exitCode = systemProcess.ExitCode; DryadLogger.LogInformation("Process exit", "Process {0} exit code {1}", DryadId, exitCode); if (exitCode == 0) { lock (syncRoot) { state = ProcessState.Completed; } } else { lock (syncRoot) { state = ProcessState.Completed; this.failed = true; } } } // // Ensure that the vertex complete event is sent to GM and that all pending properties are handled // ThreadPool.QueueUserWorkItem(new WaitCallback(ExitProcessThreadProc)); DryadLogger.LogMethodExit(); }
/// <summary> /// Initialize the endpoint addresses for each vertex host /// </summary> /// <param name="vertexEndpointAddresses">List of vertex host addresses</param> void IDryadVertexService.Initialize(StringDictionary vertexEndpointAddresses) { DryadLogger.LogMethodEntry(vertexEndpointAddresses.Count); try { this.vertexEndpointAddresses = vertexEndpointAddresses; } catch (Exception e) { DryadLogger.LogWarning("Initialize", "Operation threw exception: {0}", e.ToString()); } DryadLogger.LogMethodExit(); }
/// <summary> /// Shut down the vertex service /// </summary> /// <param name="ShutdownCode"></param> void IDryadVertexService.Shutdown(uint ShutdownCode) { DryadLogger.LogMethodEntry(ShutdownCode); try { ReplyDispatcher.ShuttingDown = true; VertexService.shutdownEvent.Set(); } catch (Exception e) { DryadLogger.LogWarning("Shutdown", "Operation threw exception: {0}", e.ToString()); } DryadLogger.LogMethodExit(); }
/// <summary> /// Make sure all pending properties get handled and vertex complete event is sent to GM /// </summary> /// <param name="obj"></param> void ExitProcessThreadProc(Object obj) { DryadLogger.LogMethodEntry(); // // Wait until all property waiters have been notified and the final // status message sent, iff the process completed successfully // do { // // Clear any thing intended for the vertex // SetAllPropertyWaiters(); lock (syncRoot) { // If nobody is waiting, AND if (propertyWaiters == 0) { // Process did not complete successfully OR // final status message has already been sent if (!Succeeded || finalStatusMessageSent) { // Then we can send the Process Exit notification break; } } } Thread.Sleep(10); } while(true); ReplyDispatcher.ProcessExited(this.graphManagerReplyUri, this.dryadProcessId, this.exitCode); // // This should never happen unless a property is requested after the vertex completed event is sent // so it's not a big deal if it does because the GM knows that the vertex is done // if (propertyWaiters > 0) { DryadLogger.LogWarning("Process exit", "Leaving thread with {0} property waiter(s).", propertyWaiters); } DryadLogger.LogMethodExit(); }
/// <summary> /// /// </summary> /// <param name="disposing"></param> private void Dispose(bool disposing) { DryadLogger.LogMethodEntry(disposing); if (!this.m_disposed) { lock (syncRoot) { if (!this.m_disposed) { if (disposing) { // Close start event handle try { processStartEvent.Close(); } catch (Exception ex) { DryadLogger.LogError(0, ex); } // Close any get/set property wait handles foreach (KeyValuePair <string, Dictionary <ulong, ManualResetEvent> > label in propertyWaitEvents) { foreach (KeyValuePair <ulong, ManualResetEvent> version in label.Value) { try { version.Value.Close(); } catch (Exception ex) { DryadLogger.LogError(0, ex); } } } propertyWaitEvents.Clear(); } m_disposed = true; } } } DryadLogger.LogMethodExit(); }
/// <summary> /// Creates a request for property update from one vertex to another (one vertex is usually GM) /// </summary> /// <param name="replyEpr">Address to send response to</param> /// <param name="infos"></param> /// <param name="blockOnLabel"></param> /// <param name="blockOnVersion"></param> /// <param name="maxBlockTime"></param> /// <param name="getPropLabel"></param> /// <param name="ProcessStatistics"></param> /// <returns>Returns success/failure of thread startup</returns> public bool SetGetProps(string replyEpr, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics) { DryadLogger.LogMethodEntry(replyEpr, this.DryadId); // // Check if graph manager is the one calling. If so, increment propertyWaiter count // if (ReplyDispatcher.IsGraphMrgUri(replyEpr)) { int n = Interlocked.Increment(ref propertyWaiters); } // // Create new property request with provided parameters and queue up request sending // PropertyRequest req = new PropertyRequest(replyEpr, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics); bool result = ThreadPool.QueueUserWorkItem(new WaitCallback(SetGetPropThreadProc), req); DryadLogger.LogMethodExit(result); return(result); }
public bool Start(string listenUri, ISchedulerHelper schedulerHelper) { DryadLogger.LogMethodEntry(listenUri); Uri baseAddress = new Uri(listenUri); try { NetTcpBinding binding = schedulerHelper.GetVertexServiceBinding(); selfHost = null; // Retry opening the service port if address is already in use int maxRetryCount = 20; // Results in retrying for ~1 min for (int retryCount = 0; retryCount < maxRetryCount; retryCount++) { try { //Step 1 of the hosting procedure: Create ServiceHost selfHost = new ServiceHost(callbackService, baseAddress); //Step 2 of the hosting procedure: Add service endpoints. ServiceEndpoint vertexEndpoint = selfHost.AddServiceEndpoint(typeof(IDryadVertexCallback), binding, Constants.vertexCallbackServiceName); ServiceThrottlingBehavior stb = new ServiceThrottlingBehavior(); stb.MaxConcurrentCalls = Constants.MaxConnections; stb.MaxConcurrentSessions = Constants.MaxConnections; selfHost.Description.Behaviors.Add(stb); //Step 3 of hosting procedure : Add a security manager selfHost.Authorization.ServiceAuthorizationManager = new DryadVertexServiceAuthorizationManager(); // Step 4 of the hosting procedure: Start the service. selfHost.Open(); break; } catch (AddressAlreadyInUseException) { if (selfHost != null) { selfHost.Abort(); selfHost = null; } // If this is the last try, dont sleep. Just rethrow exception to exit. if (retryCount < maxRetryCount - 1) { DryadLogger.LogInformation("Start Vertex Callback Service", "Address already in use. Retrying..."); System.Threading.Thread.Sleep(3000); } else { throw; } } } DryadLogger.LogInformation("Start Vertex Callback Service", "Service Host started successfully"); return(true); } catch (CommunicationException ce) { DryadLogger.LogCritical(0, ce, "Failed to start vertex callback service"); try { if (selfHost != null) { selfHost.Abort(); } } catch { } return(false); } }
/// <summary> /// Adds specified property to property wait list and waits for it. /// </summary> /// <param name="blockOnLabel">Property label to wait for</param> /// <param name="blockOnVersion">Version of property to wait for</param> /// <param name="maxBlockTime">Time to wait for property</param> /// <returns>False if property was requested but none was returned</returns> private bool BlockOnProperty(string blockOnLabel, ulong blockOnVersion, long maxBlockTime) { DryadLogger.LogMethodEntry(); // // Return true if no label is provided // if (String.IsNullOrEmpty(blockOnLabel)) { DryadLogger.LogMethodExit(true); return(true); } DryadLogger.LogInformation("Block on property", "Label {0} Version {1} maxBlockTime {2}", blockOnLabel, blockOnVersion, maxBlockTime); ProcessPropertyInfo prop = null; // // If the process already exited, don't bother adding a wait event for // this property - if it's not already set it never will be. // lock (syncRoot) { if (!exited) { // // Add this label and version to the wait events list if needed // if (propertyWaitEvents.ContainsKey(blockOnLabel) == false) { propertyWaitEvents.Add(blockOnLabel, new Dictionary <ulong, ManualResetEvent>()); } if (propertyWaitEvents[blockOnLabel].ContainsKey(blockOnVersion) == false) { propertyWaitEvents[blockOnLabel].Add(blockOnVersion, new ManualResetEvent(false)); } } else { DryadLogger.LogInformation("Block on property", "Process {0} already exited, not adding waiter", this.DryadId); } } // todo: We still may want to implement timeouts to deal with deadlocks in the service / host but it hasn't been an issue yet. //if (propertyWaitEvents[blockOnLabel][blockOnVersion].WaitOne(new TimeSpan(maxBlockTime), false)) // // Wait forever (or until process exits or is disposed) for the property to be set or interrupted // while (!exited) { try { if (propertyWaitEvents[blockOnLabel][blockOnVersion].WaitOne(100, false)) { break; } } catch (ObjectDisposedException) { DryadLogger.LogWarning("Block on property", "Process {0} disposed while waiting for label {1}, version {2}", DryadId, blockOnLabel, blockOnVersion); DryadLogger.LogMethodExit(false); return(false); } } // Did we get the property, or did the process // terminate? int index; if (TryGetProperty(blockOnLabel, out prop, out index)) { // // If a property was successfully returned, return true // if ((blockOnVersion == 0) || (prop.propertyVersion > blockOnVersion)) { DryadLogger.LogMethodExit(true); return(true); } if (state == ProcessState.Completed) { DryadLogger.LogInformation("Block on property", "Vertex completed (wait) requested version:{0} returned version:{1} of label {2}", blockOnVersion, prop.propertyVersion, blockOnLabel); DryadLogger.LogMethodExit(true); return(true); } } // // Return false if property was requested but none was found // DryadLogger.LogMethodExit(false); return(false); }
/// <summary> /// Set a property and record the version /// </summary> /// <param name="property"></param> /// <param name="newVersion"></param> private void SetProperty(ProcessPropertyInfo property, out ulong newVersion) { DryadLogger.LogMethodEntry(property.propertyLabel); ProcessPropertyInfo oldProperty = null; lock (syncRoot) { int index; if (TryGetProperty(property.propertyLabel, out oldProperty, out index)) { // // If property found in local array, then we are setting a new version of existing property // Copy the new property information into the array // oldProperty.propertyVersion++; newVersion = oldProperty.propertyVersion; if (property.propertyBlock != null && property.propertyBlock.Length > 0) { oldProperty.propertyBlock = property.propertyBlock; } oldProperty.propertyString = property.propertyString; CopyProp(oldProperty, out info.propertyInfos[index]); } else { // // if property not found in local array, then setting a new property // use version 1, unless valid value specified // if (property.propertyVersion == ulong.MaxValue || property.propertyVersion == 0) { property.propertyVersion = 1; } newVersion = property.propertyVersion; // // Create or resize the local info array as necessary and append this property // if (info.propertyInfos == null) { info.propertyInfos = new ProcessPropertyInfo[1]; } else { Array.Resize(ref info.propertyInfos, info.propertyInfos.Length + 1); } info.propertyInfos[info.propertyInfos.Length - 1] = property; } // // If there was a vertex completed event, record the latest vertex status // if (StatusMessageContainsDryadError_VertexCompleted(property.propertyLabel)) { CopyProp(property, out latestVertexStatusReceived); latestVertexStatusReceived.propertyVersion = newVersion; } // // Wake up anyone waiting for a property change by adding a new wait event for this property if needed // if (propertyWaitEvents.ContainsKey(property.propertyLabel) == false) { propertyWaitEvents.Add(property.propertyLabel, new Dictionary <ulong, ManualResetEvent>()); } // // Wake up anyone waiting for this version of the property // if (propertyWaitEvents[property.propertyLabel].ContainsKey(newVersion - 1)) { propertyWaitEvents[property.propertyLabel][newVersion - 1].Set(); } else { propertyWaitEvents[property.propertyLabel].Add(newVersion - 1, new ManualResetEvent(true)); } // // Wake up anyone waiting for any version of this property // if (newVersion > 1) { if (propertyWaitEvents[property.propertyLabel].ContainsKey(0)) { propertyWaitEvents[property.propertyLabel][0].Set(); } } } DryadLogger.LogMethodExit(); }
/// <summary> /// Called in new thread in setgetproperty service operation /// </summary> /// <param name="obj"></param> void SetGetPropThreadProc(Object obj) { DryadLogger.LogMethodEntry(DryadId); PropertyRequest r = obj as PropertyRequest; ProcessInfo infoLocal = new ProcessInfo(); ulong[] propertyVersions = null; string[] propertyLabels = null; // // Make sure process is started before continuing // if (this.State < ProcessState.Running) { try { processStartEvent.WaitOne(); } catch (ObjectDisposedException ex) { // The process was cancelled and released before it started running, just return if (exited) { DryadLogger.LogInformation("SetGetProp Thread", "Process {0} cancelled or exited before starting.", this.DryadId); } else { DryadLogger.LogError(0, ex); } DryadLogger.LogMethodExit(); return; } } // // Use status_pending if running, vertex initialization failure if process is failed and process exit code otherwise // infoLocal.processStatus = 0x103; // WinNT.h STATUS_PENDING infoLocal.processState = state; if (state == ProcessState.Running) { infoLocal.exitCode = 0x103; // WinNT.h STATUS_PENDING } else if (failed) { infoLocal.exitCode = Constants.DrError_VertexError; } else if (cancelled) { infoLocal.exitCode = Constants.DrError_VertexReceivedTermination; // DryadError_VertexReceivedTermination } else { infoLocal.exitCode = (uint)systemProcess.ExitCode; } // // Record specified properties and update versions - wakes up anyone waiting for property changes // SetProperties(r.infos, out propertyLabels, out propertyVersions); // // Try to get property update // if (BlockOnProperty(r.blockOnLabel, r.blockOnVersion, r.maxBlockTime)) { // // If property update was received, update the received property information // If received property marks vertex completed, record that // if (r.getPropLabel != null && r.getPropLabel.Length > 0) { lock (syncRoot) { infoLocal.propertyInfos = new ProcessPropertyInfo[1]; int index; if (TryGetProperty(r.getPropLabel, out infoLocal.propertyInfos[0], out index) == false) { DryadLogger.LogError(0, null, "Failed to get property for label {0}", r.getPropLabel); } if (StatusMessageContainsDryadError_VertexCompleted(infoLocal.propertyInfos[0].propertyLabel)) { CopyProp(infoLocal.propertyInfos[0], out latestVertexStatusSent); } } } // // If request asks for statistics on vertex process, get them // if (r.ProcessStatistics) { if (GetStatistics(out infoLocal.processStatistics) == false) { DryadLogger.LogError(0, null, "Failed to get vertex statistics"); } } } // // Try to report property change, if unsuccessful, kill the running vertex host process // if (!ReplyDispatcher.SetGetPropsComplete(r.replyUri, systemProcess, dryadProcessId, infoLocal, propertyLabels, propertyVersions)) { try { systemProcess.Kill(); } catch (InvalidOperationException /* unused ioe */) { // The process has already exited // -or- // There is no process associated with this Process object. } catch (Exception eInner) { // // all other exceptions // DryadLogger.LogError(0, eInner, "Exception calling back to '{0}'", r.replyUri); } } // // If a property was handled from the graph manager, decrement the waiter count // if (ReplyDispatcher.IsGraphMrgUri(r.replyUri)) { int n = Interlocked.Decrement(ref propertyWaiters); DryadLogger.LogInformation("SetGetProp Thread", "Process {0} propertyWaiters = {1}", DryadId, n); } lock (syncRoot) { // // If vertex process has exited, and sending vertex completed event, we can stop worrying // if (!finalStatusMessageSent) { if (latestVertexStatusSent != null) { if (!String.IsNullOrEmpty(latestVertexStatusSent.propertyString)) { if (latestVertexStatusSent.propertyString.Contains(string.Format(@"(0x{0:x8})", Constants.DrError_VertexCompleted))) { finalStatusMessageSent = true; } } } } } DryadLogger.LogMethodExit(); }
/// <summary> /// Gets information about the vertex service /// </summary> /// <returns></returns> VertexStatus IDryadVertexService.CheckStatus() { DryadLogger.LogMethodEntry(); VertexStatus status = new VertexStatus(); status.serviceIsAlive = true; // // Update information about disk usage // foreach (string disk in Environment.GetLogicalDrives()) { ulong freeDiskSpaceforUser; ulong totalDiskSpace; ulong freeDiskSpace; if (NativeMethods.GetDiskFreeSpaceEx(disk, out freeDiskSpaceforUser, out totalDiskSpace, out freeDiskSpace)) { status.freeDiskSpaces.Add(disk, freeDiskSpace); } else { // // Report any errors as warnings, as this is a non-essential call // int errorCode = Marshal.GetLastWin32Error(); Exception lastex = Marshal.GetExceptionForHR(errorCode); if (lastex != null) { DryadLogger.LogWarning("Unable to get disk space information", "Disk: {0} Error: {1}", disk, lastex.Message); } else { DryadLogger.LogWarning("Unable to get disk space information", "Disk: {0} Error Code: {1}", disk, errorCode); } } } // // Update information about memory usage // NativeMethods.MEMORYSTATUSEX memStatus = new NativeMethods.MEMORYSTATUSEX(); if (NativeMethods.GlobalMemoryStatusEx(memStatus)) { status.freePhysicalMemory = memStatus.ullAvailPhys; status.freeVirtualMemory = memStatus.ullAvailVirtual; } else { // // Report any errors as warnings, as this is a non-essential call // int errorCode = Marshal.GetLastWin32Error(); Exception lastex = Marshal.GetExceptionForHR(errorCode); if (lastex != null) { DryadLogger.LogWarning("Unable to get memory information", "Error: {0}", lastex.Message); } else { DryadLogger.LogWarning("Unable to get memory information", "Error Code: {0}", errorCode); } } // // Get process info for each running vertex process // status.runningProcessCount = 0; lock (vertexProcessTable.SyncRoot) { foreach (VertexProcess vp in this.vertexProcessTable) { VertexProcessInfo vpInfo = new VertexProcessInfo(); vpInfo.DryadId = vp.DryadId; vpInfo.commandLine = vp.commandLine; vpInfo.State = vp.State; status.vps.Add(vpInfo); if (vp.State == ProcessState.Running) { status.runningProcessCount++; } } } DryadLogger.LogMethodExit(status); return(status); }