/// <summary> /// Make sure all pending properties get handled and vertex complete event is sent to GM /// </summary> /// <param name="obj"></param> void ExitProcessThreadProc(Object obj) { DryadLogger.LogMethodEntry(); // // Wait until all property waiters have been notified and the final // status message sent, iff the process completed successfully // do { // // Clear any thing intended for the vertex // SetAllPropertyWaiters(); lock (syncRoot) { // If nobody is waiting, AND if (propertyWaiters == 0) { // Process did not complete successfully OR // final status message has already been sent if (!Succeeded || finalStatusMessageSent) { // Then we can send the Process Exit notification break; } } } Thread.Sleep(10); } while(true); ReplyDispatcher.ProcessExited(this.graphManagerReplyUri, this.dryadProcessId, this.exitCode); // // This should never happen unless a property is requested after the vertex completed event is sent // so it's not a big deal if it does because the GM knows that the vertex is done // if (propertyWaiters > 0) { DryadLogger.LogWarning("Process exit", "Leaving thread with {0} property waiter(s).", propertyWaiters); } DryadLogger.LogMethodExit(); }
/// <summary> /// Creates a request for property update from one vertex to another (one vertex is usually GM) /// </summary> /// <param name="replyEpr">Address to send response to</param> /// <param name="infos"></param> /// <param name="blockOnLabel"></param> /// <param name="blockOnVersion"></param> /// <param name="maxBlockTime"></param> /// <param name="getPropLabel"></param> /// <param name="ProcessStatistics"></param> /// <returns>Returns success/failure of thread startup</returns> public bool SetGetProps(string replyEpr, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics) { DryadLogger.LogMethodEntry(replyEpr, this.DryadId); // // Check if graph manager is the one calling. If so, increment propertyWaiter count // if (ReplyDispatcher.IsGraphMrgUri(replyEpr)) { int n = Interlocked.Increment(ref propertyWaiters); } // // Create new property request with provided parameters and queue up request sending // PropertyRequest req = new PropertyRequest(replyEpr, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics); bool result = ThreadPool.QueueUserWorkItem(new WaitCallback(SetGetPropThreadProc), req); DryadLogger.LogMethodExit(result); return(result); }
/// <summary> /// Called in new thread in setgetproperty service operation /// </summary> /// <param name="obj"></param> void SetGetPropThreadProc(Object obj) { DryadLogger.LogMethodEntry(DryadId); PropertyRequest r = obj as PropertyRequest; ProcessInfo infoLocal = new ProcessInfo(); ulong[] propertyVersions = null; string[] propertyLabels = null; // // Make sure process is started before continuing // if (this.State < ProcessState.Running) { try { processStartEvent.WaitOne(); } catch (ObjectDisposedException ex) { // The process was cancelled and released before it started running, just return if (exited) { DryadLogger.LogInformation("SetGetProp Thread", "Process {0} cancelled or exited before starting.", this.DryadId); } else { DryadLogger.LogError(0, ex); } DryadLogger.LogMethodExit(); return; } } // // Use status_pending if running, vertex initialization failure if process is failed and process exit code otherwise // infoLocal.processStatus = 0x103; // WinNT.h STATUS_PENDING infoLocal.processState = state; if (state == ProcessState.Running) { infoLocal.exitCode = 0x103; // WinNT.h STATUS_PENDING } else if (failed) { infoLocal.exitCode = Constants.DrError_VertexError; } else if (cancelled) { infoLocal.exitCode = Constants.DrError_VertexReceivedTermination; // DryadError_VertexReceivedTermination } else { infoLocal.exitCode = (uint)systemProcess.ExitCode; } // // Record specified properties and update versions - wakes up anyone waiting for property changes // SetProperties(r.infos, out propertyLabels, out propertyVersions); // // Try to get property update // if (BlockOnProperty(r.blockOnLabel, r.blockOnVersion, r.maxBlockTime)) { // // If property update was received, update the received property information // If received property marks vertex completed, record that // if (r.getPropLabel != null && r.getPropLabel.Length > 0) { lock (syncRoot) { infoLocal.propertyInfos = new ProcessPropertyInfo[1]; int index; if (TryGetProperty(r.getPropLabel, out infoLocal.propertyInfos[0], out index) == false) { DryadLogger.LogError(0, null, "Failed to get property for label {0}", r.getPropLabel); } if (StatusMessageContainsDryadError_VertexCompleted(infoLocal.propertyInfos[0].propertyLabel)) { CopyProp(infoLocal.propertyInfos[0], out latestVertexStatusSent); } } } // // If request asks for statistics on vertex process, get them // if (r.ProcessStatistics) { if (GetStatistics(out infoLocal.processStatistics) == false) { DryadLogger.LogError(0, null, "Failed to get vertex statistics"); } } } // // Try to report property change, if unsuccessful, kill the running vertex host process // if (!ReplyDispatcher.SetGetPropsComplete(r.replyUri, systemProcess, dryadProcessId, infoLocal, propertyLabels, propertyVersions)) { try { systemProcess.Kill(); } catch (InvalidOperationException /* unused ioe */) { // The process has already exited // -or- // There is no process associated with this Process object. } catch (Exception eInner) { // // all other exceptions // DryadLogger.LogError(0, eInner, "Exception calling back to '{0}'", r.replyUri); } } // // If a property was handled from the graph manager, decrement the waiter count // if (ReplyDispatcher.IsGraphMrgUri(r.replyUri)) { int n = Interlocked.Decrement(ref propertyWaiters); DryadLogger.LogInformation("SetGetProp Thread", "Process {0} propertyWaiters = {1}", DryadId, n); } lock (syncRoot) { // // If vertex process has exited, and sending vertex completed event, we can stop worrying // if (!finalStatusMessageSent) { if (latestVertexStatusSent != null) { if (!String.IsNullOrEmpty(latestVertexStatusSent.propertyString)) { if (latestVertexStatusSent.propertyString.Contains(string.Format(@"(0x{0:x8})", Constants.DrError_VertexCompleted))) { finalStatusMessageSent = true; } } } } } DryadLogger.LogMethodExit(); }
/// <summary> /// Asynchronously called on start command /// </summary> /// <param name="obj"></param> void StartProcessThreadProc(Object obj) { ManualResetEvent serviceInitializedEvent = obj as ManualResetEvent; bool started = false; try { // // Wait for service initialization // serviceInitializedEvent.WaitOne(); if (ExecutionHelper.InitializeForProcessExecution(dryadProcessId, Environment.GetEnvironmentVariable("XC_RESOURCEFILES"))) { // // Vertex working directory configured successfully, start the vertex host // environment.Add(Constants.vertexSvcLocalAddrEnvVar, localAddress); ProcessStartInfo startInfo = new ProcessStartInfo(); startInfo.CreateNoWindow = true; startInfo.UseShellExecute = false; startInfo.WorkingDirectory = ProcessPathHelper.ProcessPath(dryadProcessId); //YARN Debugging //var procEnvVarKeys = startInfo.EnvironmentVariables.Keys; //foreach (string key in procEnvVarKeys) //{ // DryadLogger.LogInformation("StartProcess", "key: '{0}' value: '{1}'", key, startInfo.EnvironmentVariables[key]); //} string[] args = commandLine.Split(' '); string arg = ""; for (int i = 1; i < args.Length; i++) { arg += args[i] + " "; } // // Use either FQ path or path relative to job path // if (Path.IsPathRooted(args[0])) { startInfo.FileName = args[0]; } else { startInfo.FileName = Path.Combine(ProcessPathHelper.JobPath, args[0]); } DryadLogger.LogInformation("StartProcess", "FileName: '{0}'", startInfo.FileName); // // Add environment variable to vertex host process // startInfo.Arguments = arg; foreach (DictionaryEntry entry in environment) { string key = entry.Key.ToString(); if (key == null || startInfo.EnvironmentVariables.ContainsKey(key)) { DryadLogger.LogInformation("StartProcess", "Attempting to add existing key '{0}' with value '{1}'", entry.Key, entry.Value); } else { startInfo.EnvironmentVariables.Add(key, entry.Value.ToString()); } } lock (syncRoot) { // // After taking lock, start the vertex host process and set up exited event handler // if (cancelled) { // If we've already been canceled, don't start the process DryadLogger.LogInformation("Process start", "Not starting process {0} due to receipt of cancellation", DryadId); return; } else { systemProcess = new Process(); systemProcess.StartInfo = startInfo; systemProcess.EnableRaisingEvents = true; systemProcess.Exited += new EventHandler(Process_Exited); Console.WriteLine("Process start - Vertex host process starting"); started = systemProcess.Start(); Console.WriteLine("Process start - Vertex host process started"); if (started) { DryadLogger.LogInformation("Process start", "Vertex host process started"); state = ProcessState.Running; } else { DryadLogger.LogError(0, null, "Vertex host process failed to start"); } } } } else { DryadLogger.LogError(0, null, "Initialization failed"); } } catch (Exception e) { DryadLogger.LogError(0, e, "Error starting vertex"); } if (started) { // // Notify Graph Manager that process started if successful // bool success = ReplyDispatcher.FireStateChange(this.graphManagerReplyUri, this.dryadProcessId, ProcessState.Running); if (!success) { // // Graph manager doesn't know we started and we have no way to tell it, so it's // best to just fail the vertex service task and let the job manager inform the graph manager // VertexService.Surrender(new Exception("Unable to communicate with graph manager.")); } } else { // // Otherwise, notify GM that process has failed // lock (syncRoot) { // If we've already been canceled, we don't need to change state or record the initialization failure if (!cancelled) { state = ProcessState.Completed; this.failed = true; exitCode = unchecked ((int)Constants.DrError_VertexInitialization); // DryadError_VertexInitialization } } if (failed) // This also means we weren't canceled { // Notify the Graph Manager that the process failed to start Process_Exited(this, null); } } // // Make sure process start event is set // processStartEvent.Set(); }