Ejemplo n.º 1
0
        /// <summary>
        /// Removes reference to a vertex process
        /// </summary>
        /// <param name="processId">process id to forget</param>
        void IDryadVertexService.ReleaseProcess(int processId)
        {
            DryadLogger.LogMethodEntry(processId);
            VertexProcess vp = null;

            try
            {
                vp = FindByDryadId(processId);
                if (vp != null)
                {
                    vertexProcessTable.Remove(vp);
                    vp.Dispose();
                }
                else
                {
                    DryadLogger.LogWarning("Release Process", "Unknown process id {0}", processId);
                }
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Release Process", "Operation threw exception: {0}", e.ToString());
            }

            DryadLogger.LogMethodExit();
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Update properties
        /// </summary>
        /// <param name="replyEpr">callback URI</param>
        /// <param name="processId">vertex process id</param>
        /// <param name="infos">property information</param>
        /// <param name="blockOnLabel">property update label</param>
        /// <param name="blockOnVersion">property update version</param>
        /// <param name="maxBlockTime">maximum time to wait for update</param>
        /// <param name="getPropLabel">property to get</param>
        /// <param name="ProcessStatistics">vertex host process statistics</param>
        /// <returns>success/failure of property update</returns>
        bool IDryadVertexService.SetGetProps(string replyEpr, int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics)
        {
            DryadLogger.LogMethodEntry(replyEpr, processId);
            bool success = false;

            try
            {
                // Get the vertex process ID
                VertexProcess vp = FindByDryadId(processId);
                if (vp != null)
                {
                    success = vp.SetGetProps(replyEpr, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics);
                }
                else
                {
                    DryadLogger.LogError(0, null, "Failed to set / get process properties: Unknown process id {0}", processId);
                }
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Set Or Get Process Properties", "Operation threw exception: {0}", e.ToString());
                throw new FaultException <VertexServiceError>(new VertexServiceError("SetGetProps", e.ToString()));
            }

            DryadLogger.LogMethodExit(success);
            return(success);
        }
Ejemplo n.º 3
0
        bool ISchedulerHelper.WaitForTasksReady()
        {
            // The basic strategy is to wait for the maximum number of vertex tasks which is
            // practical. Start by waiting for AllocatedNodes.Count.  As tasks fail or are cancelled,
            // decrement the number of tasks to wait for until we drop below Min at which time the
            // scheduler will end the job. Also, if tasks are rerun, increment the number of tasks to wait for.
            do
            {
                // Event set by the Task Monitor Thread when it finishes processes a batch of changes.
                m_taskChangeEvt.WaitOne();

                // Don't want OnVertexChangeHandler updating these counts while we're checking them
                lock (this)
                {
                    DryadLogger.LogInformation("Wait for vertex tasks",
                                               "{0} tasks are running, waiting for at least {1} before starting",
                                               m_runningTasks, m_startNodes);
                    if (m_runningTasks >= m_startNodes)
                    {
                        // We have enough running tasks to start
                        DryadLogger.LogDebug("Wait for vertex tasks",
                                             "Sufficient number of tasks transitioned to running to begin: {0} running tasks",
                                             m_runningTasks);
                        return(true);
                    }
                }
            } while (true);
        }
Ejemplo n.º 4
0
        public void TransitionToRunning(object state)
        {
            DryadLogger.LogDebug("Change State", "Transitioning to Running with current state {0} for process {1}", this.m_currentState.ToString(), this.m_id);

            try
            {
                // In rare cases (such as a cancelled duplicate), the GM may close the handle to the process while it is transitioning to running.
                // This results in Dispose being called on this process, which closes the m_assignedToNode handle.
                // In this case, we want to catch the exception and log it, but do nothing else, since the GM is done with this process.
                if (m_assignedToNodeEvent.WaitOne(new TimeSpan(0, 0, 10), false))
                {
                    DryadLogger.LogDebug("Change State", "Successfully waited for transition to {0} for process {1}", this.m_currentState.ToString(), this.m_id);
                }
                else
                {
                    DryadLogger.LogWarning("Change State", "Timed out waiting for transition to AssignedToNode for process {0}", this.m_id);
                    // We want to fire the state change anyway or else we'll get a zombie process.
                    // The GM will handle the transition, it just may cause a delay.
                }
                ChangeState(ProcessState.Running);
            }
            catch (ObjectDisposedException ex)
            {
                DryadLogger.LogError(0, ex, "Process handle was closed while waiting for transition to assigned to node");
            }
        }
Ejemplo n.º 5
0
        public void SetIdAndVersion(string commandLine)
        {
            bool parsed = false;

            string[] args = commandLine.Split(cmdLineSeparator, StringSplitOptions.RemoveEmptyEntries);
            if (args != null)
            {
                if (args.Length == 6)
                {
                    lock (SyncRoot)
                    {
                        if (Int32.TryParse(args[4], out m_graphManagerId))
                        {
                            if (Int32.TryParse(args[5], out m_graphManagerVersion))
                            {
                                parsed = true;
                            }
                        }
                    }
                }
            }

            if (!parsed)
            {
                DryadLogger.LogWarning("Set Vertex Id And Version", "Failed to parse vertex command line: {0}", commandLine);
            }
        }
Ejemplo n.º 6
0
        private void Dispose(bool disposing)
        {
            if (!m_disposed)
            {
                if (disposing)
                {
                    DryadLogger.LogInformation("Dispose Process", "Releasing resources for process id {0}", this.m_id);

                    this.m_assignedToNodeEvent.Close();

                    foreach (KeyValuePair <ProcessState, List <ManualResetEvent> > kvp in m_stateChangeWaiters)
                    {
                        foreach (ManualResetEvent e in kvp.Value)
                        {
                            try
                            {
                                e.Close();
                            }
                            catch (Exception ex)
                            {
                                DryadLogger.LogError(0, ex);
                            }
                        }
                    }
                }
                m_disposed = true;
            }
        }
Ejemplo n.º 7
0
        public VertexStatus CheckStatus()
        {
            for (int index = 0; index < MaxRetries; index++)
            {
                try
                {
                    if (!Faulted)
                    {
                        return(this.m_client.CheckStatus());
                    }
                    break;
                }
                catch (Exception e)
                {
                    DryadLogger.LogError(0, e, "node '{0}'", m_nodeName);
                    if (!SafeOpenConnection())
                    {
                        break;
                    }
                }
            }

            RaiseFaultedEvent();

            VertexStatus s = new VertexStatus();

            s.serviceIsAlive = false;
            return(s);
        }
Ejemplo n.º 8
0
        /// <summary>
        /// Cancels the vertex process with the provided id
        /// </summary>
        /// <param name="processId">vertex process id</param>
        void IDryadVertexService.CancelScheduleProcess(int processId)
        {
            VertexProcess vp = null;

            DryadLogger.LogMethodEntry(processId);

            try
            {
                vp = FindByDryadId(processId);
                if (vp != null)
                {
                    vp.Cancel(false);
                }
                else
                {
                    DryadLogger.LogWarning("Cancel Process", "Unknown process id {0}", processId);
                }
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Cancel Process", "Operation threw exception: {0}", e.ToString());
            }

            DryadLogger.LogMethodExit();
        }
Ejemplo n.º 9
0
 /// <summary>
 /// Constructor - called when service first hosted
 /// </summary>
 public VertexService()
 {
     DryadLogger.LogMethodEntry();
     this.vertexProcessTable = new SynchronizedCollection <VertexProcess>();
     System.Threading.ThreadPool.QueueUserWorkItem(new WaitCallback(InitializationThreadProc));
     DryadLogger.LogMethodExit();
 }
Ejemplo n.º 10
0
        void ISchedulerHelper.StopTaskMonitorThread()
        {
            DryadLogger.LogMethodEntry();
            bool wait = false;

            if (m_taskMonitorThreadRunning)
            {
                lock (m_lock)
                {
                    if (m_taskMonitorThreadRunning)
                    {
                        m_threadStopEvt.Set();
                        wait = true;
                    }
                }
            }

            m_appMaster.Finish(true);

            if (wait)
            {
                try
                {
                    m_taskMonitorThread.Join();
                }
                catch (Exception e)
                {
                    DryadLogger.LogError(0, e, "Failed to wait for task monitor thread to stop.");
                }
            }
            DryadLogger.LogMethodExit();
        }
Ejemplo n.º 11
0
        public void SetGetPropsComplete(ProcessInfo info, string[] propertyLabels, ulong[] propertyVersions)
        {
            lock (SyncRoot)
            {
                // For the Set part
                if (propertyLabels != null && propertyVersions != null)
                {
                    for (int i = 0; i < propertyLabels.Length; i++)
                    {
                        if (m_propertyListeners.ContainsKey(propertyLabels[i]))
                        {
                            List <ulong> versionsToRemove = new List <ulong>();
                            foreach (KeyValuePair <ulong, GetSetPropertyEventHandler> entry in m_propertyListeners[propertyLabels[i]])
                            {
                                if (entry.Key <= propertyVersions[i] || entry.Key == ulong.MaxValue)
                                {
                                    DryadLogger.LogDebug("SetGetProsComplete", "Set complete - m_id: {0} state: {1}, label: {2}", m_id, info.processState, propertyLabels[i]);
                                    XComputeProcessGetSetPropertyEventArgs e = new XComputeProcessGetSetPropertyEventArgs(m_id, info, propertyVersions);
                                    entry.Value(this, e);

                                    versionsToRemove.Add(entry.Key);
                                }
                            }
                            foreach (ulong version in versionsToRemove)
                            {
                                m_propertyListeners[propertyLabels[i]].Remove(version);
                            }
                        }
                    }
                }

                // For the Get part
                if (info != null && info.propertyInfos != null)
                {
                    foreach (ProcessPropertyInfo propInfo in info.propertyInfos)
                    {
                        if (m_propertyListeners.ContainsKey(propInfo.propertyLabel))
                        {
                            List <ulong> versionsToRemove = new List <ulong>();
                            foreach (KeyValuePair <ulong, GetSetPropertyEventHandler> entry in m_propertyListeners[propInfo.propertyLabel])
                            {
                                if (entry.Key <= propInfo.propertyVersion || entry.Key == ulong.MaxValue)
                                {
                                    DryadLogger.LogDebug("SetGetProsComplete", "Get complete - m_id: {0} state: {1}, label: {2}", m_id, info.processState, propInfo.propertyLabel);

                                    XComputeProcessGetSetPropertyEventArgs e = new XComputeProcessGetSetPropertyEventArgs(m_id, info, propertyVersions);
                                    entry.Value(this, e);

                                    versionsToRemove.Add(entry.Key);
                                }
                            }
                            foreach (ulong version in versionsToRemove)
                            {
                                m_propertyListeners[propInfo.propertyLabel].Remove(version);
                            }
                        }
                    }
                }
            }
        }
Ejemplo n.º 12
0
 bool ISchedulerHelper.StartTaskMonitorThread()
 {
     // We only want to have one of these threads running, in case we get called more than once
     if (m_taskMonitorThreadRunning == false)
     {
         lock (m_lock)
         {
             if (m_taskMonitorThreadRunning == false)
             {
                 ((ISchedulerHelper)this).OnVertexChange += new VertexChangeEventHandler(OnVertexChangeHandler);
                 try
                 {
                     m_taskMonitorThread = new Thread(new ThreadStart(TaskMonitorThread));
                     m_taskMonitorThread.Start();
                     m_taskMonitorThreadRunning = true;
                     return(true);
                 }
                 catch (Exception e)
                 {
                     DryadLogger.LogCritical(0, e, "Failed to start task monitoring thread");
                     return(false);
                 }
             }
         }
     }
     return(true);
 }
Ejemplo n.º 13
0
        public YarnSchedulerHelper()
        {
            // init the DryadLogger, just to make sure
            DryadLogger.Start("xcompute.log");
            m_taskUpdateQueue = new BlockingCollection <VertexTask>();

            // if we are not running in a vertex, then init the GM
            string jmString = Environment.GetEnvironmentVariable(Constants.jobManager);

            if (String.IsNullOrEmpty(jmString))
            {
                m_minNodes   = int.Parse(Environment.GetEnvironmentVariable("MINIMUM_COMPUTE_NODES"));
                m_maxNodes   = int.Parse(Environment.GetEnvironmentVariable("MAXIMUM_COMPUTE_NODES"));
                m_startNodes = m_minNodes;

                m_vertices = new VertexTask[JobMaxNodes + 2];
                DryadLogger.LogInformation("YarnSchedulerHelper()", "Initializing JAVA GM");
                DryadLogger.LogInformation("YarnSchedulerHelper()", "m_maxNodes: {0}", m_maxNodes);
                AMInstance.RegisterGMCallback(new UpdateProcessState(QueueYarnUpdate));
                ((ISchedulerHelper)this).OnVertexChange += new VertexChangeEventHandler(OnVertexChangeHandler);
                m_appMaster = new AMInstance();
            }
            else
            {
                m_vertices = new VertexTask[JobMaxNodes + 2];
                DryadLogger.LogInformation("YarnSchedulerHelper()", "Not initializing JAVA GM");
            }
        }
Ejemplo n.º 14
0
        public void ProcessYarnUpdate(VertexTask v)
        {
            DryadLogger.LogInformation("ProcessYarnUpdate", "Task {0} on node {1} is in state {2}", v.Id, v.Node,
                                       v.State);
            VertexChangeEventArgs e = new VertexChangeEventArgs(v.Id);

            e.NewNode         = v.Node;
            e.NewState        = YarnTaskStateToVertexTaskState(v.State);
            e.NewRequeueCount = v.RequeueCount;

            if (m_vertices[v.Id] != null)
            {
                e.OldNode         = m_vertices[v.Id].Node;
                e.OldState        = YarnTaskStateToVertexTaskState(m_vertices[v.Id].State);
                e.OldRequeueCount = m_vertices[v.Id].RequeueCount;
            }

            if (e.NewRequeueCount != e.OldRequeueCount)
            {
                DryadLogger.LogInformation("ProcessYarnUpdate", "Task {0} requeue count changed from {1} to {2}",
                                           v.Id, e.OldRequeueCount, e.NewRequeueCount);
            }

            // Update current vertex state
            m_vertices[v.Id] = v;
            m_vertexChangeEvent(this, e);
            //m_taskChangeEvt.Set();
        }
Ejemplo n.º 15
0
        public bool SetGetProps(string replyUri, int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics)
        {
            bool faultDispatcher = true;

            for (int numRetries = 0; numRetries < MaxRetries; numRetries++)
            {
                try
                {
                    if (!Faulted)
                    {
                        return(this.m_client.SetGetProps(replyUri, processId, infos, blockOnLabel, blockOnVersion, maxBlockTime, getPropLabel, ProcessStatistics));
                    }
                    return(false);
                }
                catch (FaultException <UnknownProcessError> )
                {
                    DryadLogger.LogWarning("Set Get Process Properties", "Attempt to get or set properties for unknown process {0} on node {1}", processId, this.m_nodeName);
                    faultDispatcher = false;
                    break;
                }
                catch (FaultException <VertexServiceError> vse)
                {
                    DryadLogger.LogWarning("Set Get Process Properties", "Error setting or getting properties for process {0} on node {1}: {2}", processId, this.m_nodeName, vse.Reason);
                    faultDispatcher = false;
                    break;
                }
                catch (TimeoutException te)
                {
                    DryadLogger.LogWarning("Set Get Process Properties", "Timeout communicating with vertex service for process {0} on node {1}: {2}", processId, this.m_nodeName, te.ToString());
                    if (!SafeOpenConnection())
                    {
                        faultDispatcher = true;
                        break;
                    }
                }
                catch (CommunicationException ce)
                {
                    DryadLogger.LogWarning("Set Get Process Properties", "Error communicating with vertex service for process {0} on node {1}: {2}", processId, this.m_nodeName, ce.ToString());
                    if (!SafeOpenConnection())
                    {
                        faultDispatcher = true;
                        break;
                    }
                }
                catch (Exception e)
                {
                    DryadLogger.LogError(0, e, "Error calling SetGetProps for process {0} on node {1}", processId, m_nodeName);
                    faultDispatcher = false;
                    break;
                }
            }

            if (faultDispatcher)
            {
                RaiseFaultedEvent();
            }
            return(false);
        }
Ejemplo n.º 16
0
        //
        // Asynchronously start vertex process
        //
        public bool Start(ManualResetEvent serviceInitializedEvent)
        {
            DryadLogger.LogMethodEntry(this.DryadId);

            bool result = ThreadPool.QueueUserWorkItem(new WaitCallback(StartProcessThreadProc), serviceInitializedEvent);

            DryadLogger.LogMethodExit(result);
            return(result);
        }
        /// <summary>
        /// Check whether current operation context should be allowed access
        /// </summary>
        /// <param name="operationContext">Current operation context</param>
        /// <returns>true = allowed</returns>
        protected override bool CheckAccessCore(OperationContext operationContext)
        {
            //TODO: Put logging information to appropriate channels when available.

            //
            // Fail if context is annonymous
            //
            if (operationContext.ServiceSecurityContext.IsAnonymous)
            {
                DryadLogger.LogError(0, null, "Vertex authentication failed : Service security context is anonymous.");
                return(false);
            }

            //
            // Get identity used in current context
            //
            WindowsIdentity callerIdentity = operationContext.ServiceSecurityContext.WindowsIdentity;

            if (callerIdentity == null)
            {
                //
                // Fail if identity is not set
                //
                DryadLogger.LogError(0, null, "Vertex authentication failed : Caller identity is null.");
                return(false);
            }
            else if (callerIdentity.IsAnonymous)
            {
                //
                // Fail if identity is anonymous
                //
                DryadLogger.LogError(0, null, "Vertex authentication failed : Caller identity is anonymous.");
                return(false);
            }
            else if (!callerIdentity.IsAuthenticated)
            {
                //
                // Fail if identity is not authenticated
                //
                DryadLogger.LogError(0, null, "Vertex authentication failed : Caller identity is not authenticated.");
                return(false);
            }

            //
            // If operation context has same user as vertex service, then allow, otherwise fail.
            //
            if (this.currentIdentity.User == callerIdentity.User)
            {
                return(true);
            }
            else
            {
                DryadLogger.LogError(0, null, "Vertex authentication failed : Current identity is {0}, caller identity is {1}", this.currentIdentity.Name, callerIdentity.Name);
            }

            return(false);
        }
Ejemplo n.º 18
0
        public SchedulingResult EndScheduleProcess(IAsyncResult asyncResult)
        {
            // We don't want to retry the async end operation - if it fails retry
            // the whole scheduling operation

            try
            {
                if (!Faulted)
                {
                    if (this.m_client.EndScheduleProcess(asyncResult))
                    {
                        return(SchedulingResult.Success);
                    }
                    else
                    {
                        return(SchedulingResult.Failure);
                    }
                }
                else
                {
                    return(SchedulingResult.Failure);
                }
            }
            catch (FaultException <VertexServiceError> vse)
            {
                DryadLogger.LogWarning("Schedule Process", "Error completing schedule process {0} on node {1}: {2}", this.m_currentProcess.Id, this.m_nodeName, vse.Reason);
                return(SchedulingResult.Failure);
            }
            catch (TimeoutException te)
            {
                DryadLogger.LogWarning("Schedule Process", "Timeout communicating with vertex service for process {0} on node {1}: {2}", this.m_currentProcess.Id, this.m_nodeName, te.ToString());
            }
            catch (CommunicationException ce)
            {
                DryadLogger.LogWarning("Schedule Process", "Error communicating with vertex service for process {0} on node {1}: {2}", this.m_currentProcess.Id, this.m_nodeName, ce.ToString());
            }
            catch (Exception e)
            {
                DryadLogger.LogError(0, e, "Error calling EndScheduleProcess for process {0} on node {0}", this.m_currentProcess.Id, m_nodeName);
                return(SchedulingResult.Failure);
            }

            // If we make it here, then we need to retry the scheduling operation
            if (SafeOpenConnection())
            {
                // ScheduleProcess manages the retry count and returns false if it is exceeded
                DryadLogger.LogDebug("Schedule Process", "Communication error: retrying process {0} on node {1}", this.m_currentProcess.Id, this.m_nodeName);
                if (ScheduleProcess(m_currentReplyUri, m_currentProcess, m_currentAsyncCallback))
                {
                    return(SchedulingResult.Pending);
                }
            }

            // SafeOpenConnection failed or retry count exceeded - fault the dispatcher.
            DryadLogger.LogWarning("Schedule Process", "Connection failed to node {0}", this.m_nodeName);
            return(SchedulingResult.CommunicationError);
        }
Ejemplo n.º 19
0
 /// <summary>
 /// Fail the vertex service task
 /// </summary>
 internal static void Surrender(Exception ex)
 {
     DryadLogger.LogMethodEntry();
     ReplyDispatcher.ShuttingDown   = true;
     VertexService.internalShutdown = true;
     VertexService.ShutdownReason   = ex;
     VertexService.shutdownEvent.Set();
     DryadLogger.LogMethodExit();
 }
Ejemplo n.º 20
0
        /// <summary>
        ///
        /// </summary>
        public void Dispose()
        {
            DryadLogger.LogMethodEntry(this.DryadId);

            Dispose(true);
            GC.SuppressFinalize(this);

            DryadLogger.LogMethodExit();
        }
Ejemplo n.º 21
0
        private void TaskMonitorThread()
        {
            TimeSpan pollInterval    = TimeSpan.FromSeconds(1);
            TimeSpan maxPollInterval = TimeSpan.FromSeconds(16);

            // The main loop.  Each iteration polls for task changes.
            while (true)
            {
                bool     foundUpdate   = false;
                DateTime loopStartTime = DateTime.Now;
                //
                // Process change results from blocking queue
                //
                do
                {
                    VertexTask v = null;
                    if (m_taskUpdateQueue.TryTake(out v, pollInterval))
                    {
                        foundUpdate = true;
                        ProcessYarnUpdate(v);
                    }
                } while ((DateTime.Now - loopStartTime) < pollInterval);

                if (foundUpdate)
                {
                    // Notify WaitForTasksReady once for each polling cycle
                    // so that it gets all the changes in one batch
                    m_taskChangeEvt.Set();
                }

                // Check to see if we've been told to stop.
                // Timeout after pollInterval.
                // TODO: For better shutdown perf, we may want to check this at other places
                // or just kill the thread - but this provides a more graceful exit.
                if (m_threadStopEvt.WaitOne(pollInterval, true))
                {
                    m_taskMonitorThreadRunning = false;
                    DryadLogger.LogInformation("Task Monitoring Thread", "Received shutdown event");
                    return;
                }

                // Double the polling interval each iteration up to maxPollInterval
                if (pollInterval < maxPollInterval)
                {
                    double newSeconds = 2 * pollInterval.TotalSeconds;
                    if (newSeconds < maxPollInterval.TotalSeconds)
                    {
                        pollInterval = TimeSpan.FromSeconds(newSeconds);
                    }
                    else
                    {
                        pollInterval = maxPollInterval;
                    }
                }
            }
        }
Ejemplo n.º 22
0
        /// <summary>
        /// Attempt to call SetGetPropsComplete on specified WCF service.
        /// </summary>
        /// <param name="replyUri">Service endpoint</param>
        /// <param name="systemProcess"></param>
        /// <param name="processId"></param>
        /// <param name="info"></param>
        /// <param name="propertyLabels"></param>
        /// <param name="propertyVersions"></param>
        /// <returns></returns>
        public static bool SetGetPropsComplete(string replyUri, Process systemProcess, int processId, ProcessInfo info, string[] propertyLabels, ulong[] propertyVersions)
        {
            DryadLogger.LogMethodEntry(replyUri, processId);

            bool result = false;

            VertexCallbackServiceClient client = GetClient(replyUri);

            //
            // Try to set/get properties up to numRetries times
            //
            for (int index = 0; index < numRetries; index++)
            {
                try
                {
                    //
                    // If client is null, try reopening it
                    //
                    if (client == null)
                    {
                        client = CreateClient(replyUri);
                    }

                    //
                    // Make SetGetPropsComplete WCF call, return success
                    //
                    client.SetGetPropsComplete(processId, info, propertyLabels, propertyVersions);
                    result = true;
                    break;
                }
                catch (Exception e)
                {
                    if ((IsGraphMrgUri(replyUri) == false && systemProcess.HasExited) || shuttingDown)
                    {
                        //
                        // If trying to connect to non-running vertex or job is shutting down, don't retry and report success.
                        //
                        DisposeClient(ref client);
                        return(true);
                    }
                    else
                    {
                        //
                        // If call failed and talking to GM or running vertex process, try reopening WCF client and calling again
                        //
                        client = ReopenClientForRetry(replyUri, e);
                    }
                }
            }

            //
            // If failed to connect X times, report error
            //
            DryadLogger.LogMethodExit(result);
            return(result);
        }
Ejemplo n.º 23
0
        /// <summary>
        /// Set process state to cancelled and stop the vertex host process if possible
        /// </summary>
        public void Cancel(bool suppressNotifications)
        {
            DryadLogger.LogMethodEntry(this.DryadId);

            lock (syncRoot)
            {
                if (state == ProcessState.Completed)
                {
                    // Process has already completed before cancelation made it here, do nothing
                    DryadLogger.LogInformation("Cancel process", "Process {0} has already exited", DryadId);
                    DryadLogger.LogMethodExit();
                    return;
                }
                DryadLogger.LogInformation("Cancel process", "Process {0} has not already exited", DryadId);
                state          = ProcessState.Completed;
                this.cancelled = true;
            }

            // If the process started, kill it
            if (systemProcess != null)
            {
                try
                {
                    // Killing the process will trigger Process_Exited
                    DryadLogger.LogInformation("Cancel process", "Killing system process for process id {0}", DryadId);

                    if (suppressNotifications)
                    {
                        // Remove the Exited event handler
                        systemProcess.Exited -= this.Process_Exited;
                    }
                    systemProcess.Kill();
                    DryadLogger.LogMethodExit();
                    return;
                }
                catch (Exception e)
                {
                    //
                    // Failed to kill process - log exception
                    //
                    DryadLogger.LogError(0, e, "Failed to kill system process for process id {0}", DryadId);
                }
            }
            else
            {
                DryadLogger.LogInformation("Cancel process", "Process {0} has not started yet", DryadId);
            }

            // Process was either not running or failed to die, trigger Process_Exited ourself
            if (!suppressNotifications)
            {
                Process_Exited(this, null);
            }
            DryadLogger.LogMethodExit();
        }
Ejemplo n.º 24
0
 public void FireStateChange(int processId, ProcessState newState)
 {
     try
     {
         vertexScheduler.ProcessChangeState(processId, newState);
     }
     catch (Exception e)
     {
         DryadLogger.LogError(0, e, "Failed to change state to {0} for process {1}", newState.ToString(), processId);
     }
 }
Ejemplo n.º 25
0
 public void SetGetPropsComplete(int processId, ProcessInfo info, string[] propertyLabels, ulong[] propertyVersions)
 {
     try
     {
         vertexScheduler.SetGetPropsComplete(processId, info, propertyLabels, propertyVersions);
     }
     catch (Exception e)
     {
         DryadLogger.LogError(0, e, "Failed to complete set / get properties for process {0}", processId);
     }
 }
Ejemplo n.º 26
0
        /// <summary>
        /// Schedule a vertex host process using the provided parameters
        /// </summary>
        /// <param name="replyUri">callback URI</param>
        /// <param name="processId">vertex process id</param>
        /// <param name="commandLine">vertex host command line</param>
        /// <param name="environment">vertex host environment variables</param>
        /// <returns>Success/Failure of starting vertex process thread</returns>
        bool IDryadVertexService.ScheduleProcess(string replyUri, int processId, string commandLine, StringDictionary environment)
        {
            DryadLogger.LogMethodEntry(processId, commandLine);
            bool startSuccess = false;

            Console.WriteLine("Starting process id {0} with commandLIne: '{1}", processId, commandLine);
            try
            {
                VertexProcess newProcess = null;

                lock (vertexProcessTable.SyncRoot)
                {
                    foreach (VertexProcess vp in vertexProcessTable)
                    {
                        if (vp.DryadId == processId)
                        {
                            // This means a previous call to Schedule process partially succeeded:
                            // the call made it to the service but something went wrong with the response
                            // so the GM's xcompute machinery retried the call. We can just return success
                            // for this case rather than tearing down the process and creating a new one.
                            return(true);
                        }

                        if (vp.State <= ProcessState.Running)
                        {
                            // There should be no other processes running.
                            // If there are, it means a previous communication error
                            // cause the GM to give up on this node for a while.
                            // Kill anything that's still hanging around.
                            vp.Cancel(true);
                        }
                    }

                    newProcess = new VertexProcess(
                        replyUri,
                        processId,
                        commandLine,
                        environment,
                        OperationContext.Current.Channel.LocalAddress.Uri.ToString()
                        );
                    this.vertexProcessTable.Add(newProcess);
                }

                startSuccess = newProcess.Start(initializedEvent);
            }
            catch (Exception e)
            {
                DryadLogger.LogWarning("Schedule Process", "Operation threw exception: {0}", e.ToString());
                throw new FaultException <VertexServiceError>(new VertexServiceError("ReleaseProcess", e.ToString()));
            }

            DryadLogger.LogMethodExit(startSuccess);
            return(startSuccess);
        }
Ejemplo n.º 27
0
        public void QueueYarnUpdate(int taskId, int taskState, string nodeName)
        {
            DryadLogger.LogInformation("QueueYarnUpdate", "Task {0} on node {2} is in state {3}", taskId, nodeName,
                                       taskState);
            // Set change event arguments

            YarnTaskState yTaskState = (YarnTaskState)taskState;
            VertexTask    v          = new VertexTask(taskId, nodeName, yTaskState, int.MaxValue, DateTime.UtcNow);

            m_taskUpdateQueue.Add(v);
        }
Ejemplo n.º 28
0
        /// <summary>
        /// Notify GM that vertex host process exited
        /// </summary>
        /// <param name="replyUri">GM address</param>
        /// <param name="processId">vertex process id</param>
        /// <param name="exitCode">reason for vertex host exit</param>
        /// <returns>success/failure</returns>
        public static bool ProcessExited(string replyUri, int processId, int exitCode)
        {
            DryadLogger.LogMethodEntry(replyUri, processId, exitCode);

            bool result = false;

            VertexCallbackServiceClient client = GetClient(replyUri);

            //
            // Try to notify GM that the process has exited up to numRetries times
            //
            for (int index = 0; index < numRetries; index++)
            {
                try
                {
                    //
                    // If client is null, try reopening it
                    //
                    if (client == null)
                    {
                        client = CreateClient(replyUri);
                    }

                    //
                    // Make ProcessExited WCF call, return success
                    //
                    client.ProcessExited(processId, exitCode);
                    result = true;
                    break;
                }
                catch (Exception e)
                {
                    if (shuttingDown)
                    {
                        // if shutting down, just return
                        DisposeClient(ref client);
                        return(true);
                    }
                    else
                    {
                        //
                        // If call failed, try reopening WCF client and calling again
                        //
                        client = ReopenClientForRetry(replyUri, e);
                    }
                }
            }

            //
            // If failure occurs after X retry attempts, report error
            //
            DryadLogger.LogMethodExit(result);
            return(result);
        }
Ejemplo n.º 29
0
 public void ProcessExited(int processId, int exitCode)
 {
     try
     {
         vertexScheduler.ProcessExit(processId, exitCode);
     }
     catch (Exception e)
     {
         DryadLogger.LogError(0, e, "Failed to execute process exit for process {0}", processId);
     }
 }
Ejemplo n.º 30
0
        /// <summary>
        /// Notify vertex service that the Graph Manager is done
        /// with vertex process processId
        /// </summary>
        /// <param name="processId">Process Id of the process to release</param>
        public void ReleaseProcess(int processId)
        {
            bool faultDispatcher = true;

            for (int numRetries = 0; numRetries < MaxRetries; numRetries++)
            {
                try
                {
                    if (CurrentProcess == processId)
                    {
                        m_currentProcess = null;
                    }

                    if (!Faulted)
                    {
                        this.m_client.ReleaseProcess(processId);
                    }
                    return;
                }
                // ReleaseProcess is one-way
                catch (TimeoutException te)
                {
                    DryadLogger.LogWarning("Release Process", "Timeout communicating with vertex service on node {0}: {1}", this.m_nodeName, te.ToString());
                    if (!SafeOpenConnection())
                    {
                        faultDispatcher = true;
                        break;
                    }
                }
                catch (CommunicationException ce)
                {
                    DryadLogger.LogWarning("Release Process", "Error communicating with vertex service on node {0}: {1}", this.m_nodeName, ce.ToString());
                    if (!SafeOpenConnection())
                    {
                        faultDispatcher = true;
                        break;
                    }
                }
                catch (Exception e)
                {
                    DryadLogger.LogError(0, e, "Error calling ReleaseProcess for node {0}", m_nodeName);
                    faultDispatcher = false;
                    break;
                }
            }

            if (faultDispatcher)
            {
                RaiseFaultedEvent();
            }
        }