예제 #1
0
        public void SetIdAndVersion(string commandLine)
        {
            bool parsed = false;

            string[] args = commandLine.Split(cmdLineSeparator, StringSplitOptions.RemoveEmptyEntries);
            if (args != null)
            {
                if (args.Length == 6)
                {
                    lock (SyncRoot)
                    {
                        if (Int32.TryParse(args[4], out m_graphManagerId))
                        {
                            if (Int32.TryParse(args[5], out m_graphManagerVersion))
                            {
                                parsed = true;
                            }
                        }
                    }
                }
            }

            if (!parsed)
            {
                DryadLogger.LogWarning("Set Vertex Id And Version", "Failed to parse vertex command line: {0}", commandLine);
            }
        }
예제 #2
0
        /// <summary>
        /// Cancels the vertex process with the provided id
        /// </summary>
        /// <param name="processId">vertex process id</param>
        void IDryadVertexService.CancelScheduleProcess(int processId)
        {
            VertexProcess vp = null;

            DryadLogger.LogMethodEntry(processId);

            try
            {
                vp = FindByDryadId(processId);
                if (vp != null)
                {
                    vp.Cancel(false);
                }
                else
                {
                    DryadLogger.LogWarning("Cancel Process", "Unknown process id {0}", processId);
                }
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Cancel Process", "Operation threw exception: {0}", e.ToString());
            }

            DryadLogger.LogMethodExit();
        }
예제 #3
0
        /// <summary>
        /// Update properties
        /// </summary>
        /// <param name="replyEpr">callback URI</param>
        /// <param name="processId">vertex process id</param>
        /// <param name="infos">property information</param>
        /// <param name="blockOnLabel">property update label</param>
        /// <param name="blockOnVersion">property update version</param>
        /// <param name="maxBlockTime">maximum time to wait for update</param>
        /// <param name="getPropLabel">property to get</param>
        /// <param name="ProcessStatistics">vertex host process statistics</param>
        /// <returns>success/failure of property update</returns>
        bool IDryadVertexService.SetGetProps(string replyEpr, int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics)
        {
            DryadLogger.LogMethodEntry(replyEpr, processId);
            bool success = false;

            try
            {
                // Get the vertex process ID
                VertexProcess vp = FindByDryadId(processId);
                if (vp != null)
                {
                    success = vp.SetGetProps(replyEpr, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics);
                }
                else
                {
                    DryadLogger.LogError(0, null, "Failed to set / get process properties: Unknown process id {0}", processId);
                }
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Set Or Get Process Properties", "Operation threw exception: {0}", e.ToString());
                throw new FaultException <VertexServiceError>(new VertexServiceError("SetGetProps", e.ToString()));
            }

            DryadLogger.LogMethodExit(success);
            return(success);
        }
예제 #4
0
        /// <summary>
        /// Removes reference to a vertex process
        /// </summary>
        /// <param name="processId">process id to forget</param>
        void IDryadVertexService.ReleaseProcess(int processId)
        {
            DryadLogger.LogMethodEntry(processId);
            VertexProcess vp = null;

            try
            {
                vp = FindByDryadId(processId);
                if (vp != null)
                {
                    vertexProcessTable.Remove(vp);
                    vp.Dispose();
                }
                else
                {
                    DryadLogger.LogWarning("Release Process", "Unknown process id {0}", processId);
                }
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Release Process", "Operation threw exception: {0}", e.ToString());
            }

            DryadLogger.LogMethodExit();
        }
예제 #5
0
        public void TransitionToRunning(object state)
        {
            DryadLogger.LogDebug("Change State", "Transitioning to Running with current state {0} for process {1}", this.m_currentState.ToString(), this.m_id);

            try
            {
                // In rare cases (such as a cancelled duplicate), the GM may close the handle to the process while it is transitioning to running.
                // This results in Dispose being called on this process, which closes the m_assignedToNode handle.
                // In this case, we want to catch the exception and log it, but do nothing else, since the GM is done with this process.
                if (m_assignedToNodeEvent.WaitOne(new TimeSpan(0, 0, 10), false))
                {
                    DryadLogger.LogDebug("Change State", "Successfully waited for transition to {0} for process {1}", this.m_currentState.ToString(), this.m_id);
                }
                else
                {
                    DryadLogger.LogWarning("Change State", "Timed out waiting for transition to AssignedToNode for process {0}", this.m_id);
                    // We want to fire the state change anyway or else we'll get a zombie process.
                    // The GM will handle the transition, it just may cause a delay.
                }
                ChangeState(ProcessState.Running);
            }
            catch (ObjectDisposedException ex)
            {
                DryadLogger.LogError(0, ex, "Process handle was closed while waiting for transition to assigned to node");
            }
        }
예제 #6
0
        public bool SetGetProps(string replyUri, int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics)
        {
            bool faultDispatcher = true;

            for (int numRetries = 0; numRetries < MaxRetries; numRetries++)
            {
                try
                {
                    if (!Faulted)
                    {
                        return(this.m_client.SetGetProps(replyUri, processId, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics));
                    }
                    return(false);
                }
                catch (FaultException <UnknownProcessError> )
                {
                    DryadLogger.LogWarning("Set Get Process Properties", "Attempt to get or set properties for unknown process {0} on node {1}", processId, this.m_nodeName);
                    faultDispatcher = false;
                    break;
                }
                catch (FaultException <VertexServiceError> vse)
                {
                    DryadLogger.LogWarning("Set Get Process Properties", "Error setting or getting properties for process {0} on node {1}: {2}", processId, this.m_nodeName, vse.Reason);
                    faultDispatcher = false;
                    break;
                }
                catch (TimeoutException te)
                {
                    DryadLogger.LogWarning("Set Get Process Properties", "Timeout communicating with vertex service for process {0} on node {1}: {2}", processId, this.m_nodeName, te.ToString());
                    if (!SafeOpenConnection())
                    {
                        faultDispatcher = true;
                        break;
                    }
                }
                catch (CommunicationException ce)
                {
                    DryadLogger.LogWarning("Set Get Process Properties", "Error communicating with vertex service for process {0} on node {1}: {2}", processId, this.m_nodeName, ce.ToString());
                    if (!SafeOpenConnection())
                    {
                        faultDispatcher = true;
                        break;
                    }
                }
                catch (Exception e)
                {
                    DryadLogger.LogError(0, e, "Error calling SetGetProps for process {0} on node {1}", processId, m_nodeName);
                    faultDispatcher = false;
                    break;
                }
            }

            if (faultDispatcher)
            {
                RaiseFaultedEvent();
            }
            return(false);
        }
예제 #7
0
        public SchedulingResult EndScheduleProcess(IAsyncResult asyncResult)
        {
            // We don't want to retry the async end operation - if it fails retry
            // the whole scheduling operation

            try
            {
                if (!Faulted)
                {
                    if (this.m_client.EndScheduleProcess(asyncResult))
                    {
                        return(SchedulingResult.Success);
                    }
                    else
                    {
                        return(SchedulingResult.Failure);
                    }
                }
                else
                {
                    return(SchedulingResult.Failure);
                }
            }
            catch (FaultException <VertexServiceError> vse)
            {
                DryadLogger.LogWarning("Schedule Process", "Error completing schedule process {0} on node {1}: {2}", this.m_currentProcess.Id, this.m_nodeName, vse.Reason);
                return(SchedulingResult.Failure);
            }
            catch (TimeoutException te)
            {
                DryadLogger.LogWarning("Schedule Process", "Timeout communicating with vertex service for process {0} on node {1}: {2}", this.m_currentProcess.Id, this.m_nodeName, te.ToString());
            }
            catch (CommunicationException ce)
            {
                DryadLogger.LogWarning("Schedule Process", "Error communicating with vertex service for process {0} on node {1}: {2}", this.m_currentProcess.Id, this.m_nodeName, ce.ToString());
            }
            catch (Exception e)
            {
                DryadLogger.LogError(0, e, "Error calling EndScheduleProcess for process {0} on node {0}", this.m_currentProcess.Id, m_nodeName);
                return(SchedulingResult.Failure);
            }

            // If we make it here, then we need to retry the scheduling operation
            if (SafeOpenConnection())
            {
                // ScheduleProcess manages the retry count and returns false if it is exceeded
                DryadLogger.LogDebug("Schedule Process", "Communication error: retrying process {0} on node {1}", this.m_currentProcess.Id, this.m_nodeName);
                if (ScheduleProcess(m_currentReplyUri, m_currentProcess, m_currentAsyncCallback))
                {
                    return(SchedulingResult.Pending);
                }
            }

            // SafeOpenConnection failed or retry count exceeded - fault the dispatcher.
            DryadLogger.LogWarning("Schedule Process", "Connection failed to node {0}", this.m_nodeName);
            return(SchedulingResult.CommunicationError);
        }
예제 #8
0
        /// <summary>
        /// Schedule a vertex host process using the provided parameters
        /// </summary>
        /// <param name="replyUri">callback URI</param>
        /// <param name="processId">vertex process id</param>
        /// <param name="commandLine">vertex host command line</param>
        /// <param name="environment">vertex host environment variables</param>
        /// <returns>Success/Failure of starting vertex process thread</returns>
        bool IDryadVertexService.ScheduleProcess(string replyUri, int processId, string commandLine, StringDictionary environment)
        {
            DryadLogger.LogMethodEntry(processId, commandLine);
            bool startSuccess = false;

            Console.WriteLine("Starting process id {0} with commandLIne: '{1}", processId, commandLine);
            try
            {
                VertexProcess newProcess = null;

                lock (vertexProcessTable.SyncRoot)
                {
                    foreach (VertexProcess vp in vertexProcessTable)
                    {
                        if (vp.DryadId == processId)
                        {
                            // This means a previous call to Schedule process partially succeeded:
                            // the call made it to the service but something went wrong with the response
                            // so the GM's xcompute machinery retried the call. We can just return success
                            // for this case rather than tearing down the process and creating a new one.
                            return(true);
                        }

                        if (vp.State <= ProcessState.Running)
                        {
                            // There should be no other processes running.
                            // If there are, it means a previous communication error
                            // cause the GM to give up on this node for a while.
                            // Kill anything that's still hanging around.
                            vp.Cancel(true);
                        }
                    }

                    newProcess = new VertexProcess(
                        replyUri,
                        processId,
                        commandLine,
                        environment,
                        OperationContext.Current.Channel.LocalAddress.Uri.ToString()
                        );
                    this.vertexProcessTable.Add(newProcess);
                }

                startSuccess = newProcess.Start(initializedEvent);
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Schedule Process", "Operation threw exception: {0}", e.ToString());
                throw new FaultException <VertexServiceError>(new VertexServiceError("ReleaseProcess", e.ToString()));
            }

            DryadLogger.LogMethodExit(startSuccess);
            return(startSuccess);
        }
예제 #9
0
        /// <summary>
        /// Notify vertex service that the Graph Manager is done
        /// with vertex process processId
        /// </summary>
        /// <param name="processId">Process Id of the process to release</param>
        public void ReleaseProcess(int processId)
        {
            bool faultDispatcher = true;

            for (int numRetries = 0; numRetries < MaxRetries; numRetries++)
            {
                try
                {
                    if (CurrentProcess == processId)
                    {
                        m_currentProcess = null;
                    }

                    if (!Faulted)
                    {
                        this.m_client.ReleaseProcess(processId);
                    }
                    return;
                }
                // ReleaseProcess is one-way
                catch (TimeoutException te)
                {
                    DryadLogger.LogWarning("Release Process", "Timeout communicating with vertex service on node {0}: {1}", this.m_nodeName, te.ToString());
                    if (!SafeOpenConnection())
                    {
                        faultDispatcher = true;
                        break;
                    }
                }
                catch (CommunicationException ce)
                {
                    DryadLogger.LogWarning("Release Process", "Error communicating with vertex service on node {0}: {1}", this.m_nodeName, ce.ToString());
                    if (!SafeOpenConnection())
                    {
                        faultDispatcher = true;
                        break;
                    }
                }
                catch (Exception e)
                {
                    DryadLogger.LogError(0, e, "Error calling ReleaseProcess for node {0}", m_nodeName);
                    faultDispatcher = false;
                    break;
                }
            }

            if (faultDispatcher)
            {
                RaiseFaultedEvent();
            }
        }
예제 #10
0
        /// <summary>
        /// Initialize the endpoint addresses for each vertex host
        /// </summary>
        /// <param name="vertexEndpointAddresses">List of vertex host addresses</param>
        void IDryadVertexService.Initialize(StringDictionary vertexEndpointAddresses)
        {
            DryadLogger.LogMethodEntry(vertexEndpointAddresses.Count);

            try
            {
                this.vertexEndpointAddresses = vertexEndpointAddresses;
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Initialize", "Operation threw exception: {0}", e.ToString());
            }

            DryadLogger.LogMethodExit();
        }
예제 #11
0
        /// <summary>
        /// Call Shutdown method on the vertex service and close the communication channel.
        /// After this method is called, the Dispatcher is unusable.
        /// </summary>
        /// <param name="state">uint code - reserved for future use</param>
        public void Shutdown(uint code)
        {
            for (int index = 0; index < MaxRetries; index++)
            {
                try
                {
                    if (!Faulted)
                    {
                        this.m_client.Shutdown(code);
                    }
                    return;
                }
                catch (FaultException <VertexServiceError> vse)
                {
                    DryadLogger.LogWarning("Shutdown", "Error shutting down vertex service on node {0}: {1}", this.m_nodeName, vse.Reason);
                    break;
                }
                catch (TimeoutException te)
                {
                    DryadLogger.LogWarning("Shutdown", "Timeout communicating with vertex service on node {0}: {1}", this.m_nodeName, te.ToString());
                    if (!SafeOpenConnection())
                    {
                        break;
                    }
                }
                catch (CommunicationException ce)
                {
                    DryadLogger.LogWarning("Shutdown", "Error communicating with vertex service on node {0}: {1}", this.m_nodeName, ce.ToString());
                    if (!SafeOpenConnection())
                    {
                        DryadLogger.LogWarning("Shutdown", "Failed to reopen connection to node {0}", this.m_nodeName);
                        break;
                    }
                }
                catch (Exception e)
                {
                    DryadLogger.LogWarning("Shutdown", "Exception shutting down vertex service on node {0}: {1}", this.m_nodeName, e.ToString());
                    if (!SafeOpenConnection())
                    {
                        break;
                    }
                }
            }

            // Not faulting the dispatcher here, even though the WCF connection could not be closed cleanly
            // Shutdown is only called when the graphmanger is closing, so there is no need to fault the dispatchers
            // Also avoids problems around faulting dispatchers while enumerating them
        }
예제 #12
0
        /// <summary>
        /// Shut down the vertex service
        /// </summary>
        /// <param name="ShutdownCode"></param>
        void IDryadVertexService.Shutdown(uint ShutdownCode)
        {
            DryadLogger.LogMethodEntry(ShutdownCode);

            try
            {
                ReplyDispatcher.ShuttingDown = true;
                VertexService.shutdownEvent.Set();
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Shutdown", "Operation threw exception: {0}", e.ToString());
            }

            DryadLogger.LogMethodExit();
        }
예제 #13
0
        public void Initialize(StringDictionary vertexEndpointAddresses)
        {
            bool faultDispatcher = true;

            for (int numRetries = 0; numRetries < MaxRetries; numRetries++)
            {
                try
                {
                    if (!Faulted)
                    {
                        this.m_client.Initialize(vertexEndpointAddresses);
                    }
                    return;
                }
                // Initialize is one-way
                catch (TimeoutException te)
                {
                    DryadLogger.LogWarning("Initialize", "Timeout communicating with vertex service on node {0}: {1}", this.m_nodeName, te.ToString());
                    if (!SafeOpenConnection())
                    {
                        faultDispatcher = true;
                        break;
                    }
                }
                catch (CommunicationException ce)
                {
                    DryadLogger.LogWarning("Initialize", "Error communicating with vertex service on node {0}: {1}", this.m_nodeName, ce.ToString());
                    if (!SafeOpenConnection())
                    {
                        faultDispatcher = true;
                        break;
                    }
                }
                catch (Exception e)
                {
                    DryadLogger.LogError(0, e, "Error calling Initialize for node {0}", m_nodeName);
                    faultDispatcher = false;
                    break;
                }
            }

            if (faultDispatcher)
            {
                RaiseFaultedEvent();
            }
        }
예제 #14
0
        /// <summary>
        /// Make sure all pending properties get handled and vertex complete event is sent to GM
        /// </summary>
        /// <param name="obj"></param>
        void ExitProcessThreadProc(Object obj)
        {
            DryadLogger.LogMethodEntry();

            //
            // Wait until all property waiters have been notified and the final
            // status message sent, iff the process completed successfully
            //
            do
            {
                //
                // Clear any thing intended for the vertex
                //
                SetAllPropertyWaiters();

                lock (syncRoot)
                {
                    // If nobody is waiting, AND
                    if (propertyWaiters == 0)
                    {
                        // Process did not complete successfully OR
                        // final status message has already been sent
                        if (!Succeeded || finalStatusMessageSent)
                        {
                            // Then we can send the Process Exit notification
                            break;
                        }
                    }
                }

                Thread.Sleep(10);
            } while(true);

            ReplyDispatcher.ProcessExited(this.graphManagerReplyUri, this.dryadProcessId, this.exitCode);

            //
            // This should never happen unless a property is requested after the vertex completed event is sent
            // so it's not a big deal if it does because the GM knows that the vertex is done
            //
            if (propertyWaiters > 0)
            {
                DryadLogger.LogWarning("Process exit", "Leaving thread with {0} property waiter(s).", propertyWaiters);
            }

            DryadLogger.LogMethodExit();
        }
예제 #15
0
        private void ShowProgress(string message, bool finished)
        {
            Int32 nPercent = 0;
            // Progress is incremented as active vertices complete, when they're all done
            // the GM still has to seal the output stream, which may take a nontrivial amount
            // of time, so scale to 99% until the final progress update.
            double scalingFactor = finished ? 100.0 : 99.0;

            try
            {
                nPercent = Convert.ToInt32(Convert.ToDouble(m_progressStepsCompleted) / Convert.ToDouble(m_totalProgressSteps) * scalingFactor);
                DryadLogger.LogDebug("Set Job Progress", "{0} percent complete", nPercent);
            }
            catch (OverflowException e)
            {
                DryadLogger.LogWarning("Set Job Progress", "OverflowException calculating percent complete: {0}", e.ToString());
                nPercent = 100;
            }

            if (nPercent > 100)
            {
                DryadLogger.LogWarning("Set Job Progress", "Percent complete greater than 100: {0} / {1} steps reported complete", m_progressStepsCompleted, m_totalProgressSteps);
                nPercent = 100;
            }

            try
            {
                if (message == null)
                {
                    message = String.Empty;
                }
                else if (message.Length > 80)
                {
                    // Job progress messages have max length of 80
                    message = message.Substring(0, 80);
                }
                m_schedulerHelper.SetJobProgress(nPercent, message);
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Set Job Progress", "Failed to set job progress: {0}", e.ToString());
            }
        }
예제 #16
0
 void ISchedulerHelper.SetJobProgress(int n, string message)
 {
     DryadLogger.LogWarning("SetJobProgress", "n: {0} message: {1}", n, message);
 }
예제 #17
0
        public bool ScheduleProcess(string replyUri, ScheduleProcessRequest req, AsyncCallback cb)
        {
            bool faultDispatcher = true;

            for (int numRetries = 0; numRetries < MaxRetries; numRetries++)
            {
                try
                {
                    // TODO: Why are we taking the lock in this particular case again?
                    lock (SyncRoot)
                    {
                        if (!Faulted && m_schedulingAttempts < MaxRetries)
                        {
                            m_schedulingAttempts++;

                            // Set the current process so that if the dispatcher faults we know
                            // which process to kill
                            m_currentProcess       = req;
                            m_currentReplyUri      = replyUri;
                            m_currentAsyncCallback = cb;

                            this.m_client.BeginScheduleProcess(replyUri, req.Id, req.CommandLine, req.Environment, cb, (object)this);
                            return(true);
                        }
                    }
                    return(false);
                }
                catch (FaultException <VertexServiceError> vse)
                {
                    DryadLogger.LogWarning("Schedule Process", "Error scheduling process {0} on node {1}: {2}", req.Id, this.m_nodeName, vse.Reason);
                    faultDispatcher = false;
                    break;
                }
                catch (TimeoutException te)
                {
                    DryadLogger.LogWarning("Schedule Process", "Timeout communicating with vertex service scheduling process {0} on node {1}: {2}", req.Id, this.m_nodeName, te.ToString());
                    if (!SafeOpenConnection())
                    {
                        faultDispatcher = true;
                        break;
                    }
                }
                catch (CommunicationException ce)
                {
                    DryadLogger.LogWarning("Schedule Process", "Error communicating with vertex service scheduling process {0} on node {1}: {2}", req.Id, this.m_nodeName, ce.ToString());
                    if (!SafeOpenConnection())
                    {
                        faultDispatcher = true;
                        break;
                    }
                }
                catch (Exception e)
                {
                    DryadLogger.LogError(0, e, "Error calling ScheduleProcess for process {0} on node {1}", req.Id, m_nodeName);
                    faultDispatcher = false;
                    break;
                }
            }

            if (faultDispatcher)
            {
                RaiseFaultedEvent();
            }
            return(false);
        }
예제 #18
0
        public void ChangeState(ProcessState newState)
        {
            lock (SyncRoot)
            {
                if (newState > m_currentState)
                {
                    DryadLogger.LogDebug("Change State", "Transition process {0} from state {1} to state {2}", m_id, m_currentState, newState);

                    m_currentState = newState;
                    List <ProcessState> listenersToRemove = new List <ProcessState>();
                    List <ProcessState> waitersToRemove   = new List <ProcessState>();

                    // Check for listeners / waiters for earlier states, in case a state is skipped (e.g. process failed to start)
                    foreach (ProcessState s in m_stateChangeListeners.Keys)
                    {
                        if (s <= m_currentState)
                        {
                            // Notify listeners
                            if (m_stateChangeListeners[s] != null)
                            {
                                XComputeProcessStateChangeEventArgs e = new XComputeProcessStateChangeEventArgs(m_id, m_currentState, false);
                                m_stateChangeListeners[s](this, e);
                                if (m_stateChangeTimers.ContainsKey(m_stateChangeListeners[s]))
                                {
                                    m_stateChangeTimers[m_stateChangeListeners[s]].Dispose();
                                    m_stateChangeTimers.Remove(m_stateChangeListeners[s]);
                                }
                            }
                            listenersToRemove.Add(s);
                        }
                    }
                    foreach (ProcessState s in listenersToRemove)
                    {
                        m_stateChangeListeners.Remove(s);
                    }

                    foreach (ProcessState s in m_stateChangeWaiters.Keys)
                    {
                        // Signal waiters
                        if (s <= m_currentState)
                        {
                            foreach (ManualResetEvent w in m_stateChangeWaiters[s])
                            {
                                w.Set();
                            }
                            waitersToRemove.Add(s);
                        }
                    }
                    foreach (ProcessState s in waitersToRemove)
                    {
                        foreach (ManualResetEvent e in m_stateChangeWaiters[s])
                        {
                            try
                            {
                                e.Close();
                            }
                            catch (Exception ex)
                            {
                                DryadLogger.LogError(0, ex);
                            }
                        }
                        m_stateChangeWaiters.Remove(s);
                    }

                    if (m_currentState == ProcessState.AssignedToNode)
                    {
                        m_assignedToNodeEvent.Set();
                    }
                }
                else
                {
                    DryadLogger.LogWarning("Change State", "Unexpected state change attempted for process {0}: from {1} to {2}", this.m_id, this.m_currentState.ToString(), newState.ToString());
                }
            }
        }
예제 #19
0
        /// <summary>
        /// Adds specified property to property wait list and waits for it.
        /// </summary>
        /// <param name="blockOnLabel">Property label to wait for</param>
        /// <param name="blockOnVersion">Version of property to wait for</param>
        /// <param name="maxBlockTime">Time to wait for property</param>
        /// <returns>False if property was requested but none was returned</returns>
        private bool BlockOnProperty(string blockOnLabel, ulong blockOnVersion, long maxBlockTime)
        {
            DryadLogger.LogMethodEntry();

            //
            // Return true if no label is provided
            //
            if (String.IsNullOrEmpty(blockOnLabel))
            {
                DryadLogger.LogMethodExit(true);
                return(true);
            }

            DryadLogger.LogInformation("Block on property", "Label {0} Version {1} maxBlockTime {2}", blockOnLabel, blockOnVersion, maxBlockTime);

            ProcessPropertyInfo prop = null;

            //
            // If the process already exited, don't bother adding a wait event for
            // this property - if it's not already set it never will be.
            //

            lock (syncRoot)
            {
                if (!exited)
                {
                    //
                    // Add this label and version to the wait events list if needed
                    //
                    if (propertyWaitEvents.ContainsKey(blockOnLabel) == false)
                    {
                        propertyWaitEvents.Add(blockOnLabel, new Dictionary <ulong, ManualResetEvent>());
                    }

                    if (propertyWaitEvents[blockOnLabel].ContainsKey(blockOnVersion) == false)
                    {
                        propertyWaitEvents[blockOnLabel].Add(blockOnVersion, new ManualResetEvent(false));
                    }
                }
                else
                {
                    DryadLogger.LogInformation("Block on property", "Process {0} already exited, not adding waiter", this.DryadId);
                }
            }

            // todo: We still may want to implement timeouts to deal with deadlocks in the service / host but it hasn't been an issue yet.
            //if (propertyWaitEvents[blockOnLabel][blockOnVersion].WaitOne(new TimeSpan(maxBlockTime), false))

            //
            // Wait forever (or until process exits or is disposed) for the property to be set or interrupted
            //

            while (!exited)
            {
                try
                {
                    if (propertyWaitEvents[blockOnLabel][blockOnVersion].WaitOne(100, false))
                    {
                        break;
                    }
                }
                catch (ObjectDisposedException)
                {
                    DryadLogger.LogWarning("Block on property", "Process {0} disposed while waiting for label {1}, version {2}", DryadId, blockOnLabel, blockOnVersion);
                    DryadLogger.LogMethodExit(false);
                    return(false);
                }
            }

            // Did we get the property, or did the process
            // terminate?
            int index;

            if (TryGetProperty(blockOnLabel, out prop, out index))
            {
                //
                // If a property was successfully returned, return true
                //
                if ((blockOnVersion == 0) || (prop.propertyVersion > blockOnVersion))
                {
                    DryadLogger.LogMethodExit(true);
                    return(true);
                }

                if (state == ProcessState.Completed)
                {
                    DryadLogger.LogInformation("Block on property", "Vertex completed (wait) requested version:{0} returned version:{1} of label {2}", blockOnVersion, prop.propertyVersion, blockOnLabel);
                    DryadLogger.LogMethodExit(true);
                    return(true);
                }
            }

            //
            // Return false if property was requested but none was found
            //
            DryadLogger.LogMethodExit(false);
            return(false);
        }
예제 #20
0
        /// <summary>
        /// Gets information about the vertex service
        /// </summary>
        /// <returns></returns>
        VertexStatus IDryadVertexService.CheckStatus()
        {
            DryadLogger.LogMethodEntry();
            VertexStatus status = new VertexStatus();

            status.serviceIsAlive = true;

            //
            // Update information about disk usage
            //
            foreach (string disk in Environment.GetLogicalDrives())
            {
                ulong freeDiskSpaceforUser;
                ulong totalDiskSpace;
                ulong freeDiskSpace;

                if (NativeMethods.GetDiskFreeSpaceEx(disk, out freeDiskSpaceforUser, out totalDiskSpace, out freeDiskSpace))
                {
                    status.freeDiskSpaces.Add(disk, freeDiskSpace);
                }
                else
                {
                    //
                    // Report any errors as warnings, as this is a non-essential call
                    //
                    int       errorCode = Marshal.GetLastWin32Error();
                    Exception lastex    = Marshal.GetExceptionForHR(errorCode);
                    if (lastex != null)
                    {
                        DryadLogger.LogWarning("Unable to get disk space information", "Disk: {0} Error: {1}", disk, lastex.Message);
                    }
                    else
                    {
                        DryadLogger.LogWarning("Unable to get disk space information", "Disk: {0} Error Code: {1}", disk, errorCode);
                    }
                }
            }

            //
            // Update information about memory usage
            //
            NativeMethods.MEMORYSTATUSEX memStatus = new NativeMethods.MEMORYSTATUSEX();
            if (NativeMethods.GlobalMemoryStatusEx(memStatus))
            {
                status.freePhysicalMemory = memStatus.ullAvailPhys;
                status.freeVirtualMemory  = memStatus.ullAvailVirtual;
            }
            else
            {
                //
                // Report any errors as warnings, as this is a non-essential call
                //
                int       errorCode = Marshal.GetLastWin32Error();
                Exception lastex    = Marshal.GetExceptionForHR(errorCode);
                if (lastex != null)
                {
                    DryadLogger.LogWarning("Unable to get memory information", "Error: {0}", lastex.Message);
                }
                else
                {
                    DryadLogger.LogWarning("Unable to get memory information", "Error Code: {0}", errorCode);
                }
            }

            //
            // Get process info for each running vertex process
            //
            status.runningProcessCount = 0;
            lock (vertexProcessTable.SyncRoot)
            {
                foreach (VertexProcess vp in this.vertexProcessTable)
                {
                    VertexProcessInfo vpInfo = new VertexProcessInfo();
                    vpInfo.DryadId     = vp.DryadId;
                    vpInfo.commandLine = vp.commandLine;
                    vpInfo.State       = vp.State;

                    status.vps.Add(vpInfo);

                    if (vp.State == ProcessState.Running)
                    {
                        status.runningProcessCount++;
                    }
                }
            }

            DryadLogger.LogMethodExit(status);
            return(status);
        }