Example #1
0
        public void ProcessYarnUpdate(VertexTask v)
        {
            DryadLogger.LogInformation("ProcessYarnUpdate", "Task {0} on node {1} is in state {2}", v.Id, v.Node,
                                       v.State);
            VertexChangeEventArgs e = new VertexChangeEventArgs(v.Id);

            e.NewNode         = v.Node;
            e.NewState        = YarnTaskStateToVertexTaskState(v.State);
            e.NewRequeueCount = v.RequeueCount;

            if (m_vertices[v.Id] != null)
            {
                e.OldNode         = m_vertices[v.Id].Node;
                e.OldState        = YarnTaskStateToVertexTaskState(m_vertices[v.Id].State);
                e.OldRequeueCount = m_vertices[v.Id].RequeueCount;
            }

            if (e.NewRequeueCount != e.OldRequeueCount)
            {
                DryadLogger.LogInformation("ProcessYarnUpdate", "Task {0} requeue count changed from {1} to {2}",
                                           v.Id, e.OldRequeueCount, e.NewRequeueCount);
            }

            // Update current vertex state
            m_vertices[v.Id] = v;
            m_vertexChangeEvent(this, e);
            //m_taskChangeEvt.Set();
        }
Example #2
0
        private void TaskMonitorThread()
        {
            TimeSpan pollInterval    = TimeSpan.FromSeconds(1);
            TimeSpan maxPollInterval = TimeSpan.FromSeconds(16);

            // The main loop.  Each iteration polls for task changes.
            while (true)
            {
                bool     foundUpdate   = false;
                DateTime loopStartTime = DateTime.Now;
                //
                // Process change results from blocking queue
                //
                do
                {
                    VertexTask v = null;
                    if (m_taskUpdateQueue.TryTake(out v, pollInterval))
                    {
                        foundUpdate = true;
                        ProcessYarnUpdate(v);
                    }
                } while ((DateTime.Now - loopStartTime) < pollInterval);

                if (foundUpdate)
                {
                    // Notify WaitForTasksReady once for each polling cycle
                    // so that it gets all the changes in one batch
                    m_taskChangeEvt.Set();
                }

                // Check to see if we've been told to stop.
                // Timeout after pollInterval.
                // TODO: For better shutdown perf, we may want to check this at other places
                // or just kill the thread - but this provides a more graceful exit.
                if (m_threadStopEvt.WaitOne(pollInterval, true))
                {
                    m_taskMonitorThreadRunning = false;
                    DryadLogger.LogInformation("Task Monitoring Thread", "Received shutdown event");
                    return;
                }

                // Double the polling interval each iteration up to maxPollInterval
                if (pollInterval < maxPollInterval)
                {
                    double newSeconds = 2 * pollInterval.TotalSeconds;
                    if (newSeconds < maxPollInterval.TotalSeconds)
                    {
                        pollInterval = TimeSpan.FromSeconds(newSeconds);
                    }
                    else
                    {
                        pollInterval = maxPollInterval;
                    }
                }
            }
        }
Example #3
0
        public void QueueYarnUpdate(int taskId, int taskState, string nodeName)
        {
            DryadLogger.LogInformation("QueueYarnUpdate", "Task {0} on node {2} is in state {3}", taskId, nodeName,
                                       taskState);
            // Set change event arguments

            YarnTaskState yTaskState = (YarnTaskState)taskState;
            VertexTask    v          = new VertexTask(taskId, nodeName, yTaskState, int.MaxValue, DateTime.UtcNow);

            m_taskUpdateQueue.Add(v);
        }
Example #4
0
        public void QueueYarnUpdate(int taskId, int taskState, string nodeName)
        {
            DryadLogger.LogInformation("QueueYarnUpdate", "Task {0} on node {2} is in state {3}", taskId, nodeName,
                                       taskState);
            // Set change event arguments

            YarnTaskState yTaskState = (YarnTaskState)taskState;
            VertexTask v = new VertexTask(taskId, nodeName, yTaskState, int.MaxValue, DateTime.UtcNow);
            m_taskUpdateQueue.Add(v);
        }
Example #5
0
        public void ProcessYarnUpdate(VertexTask v)
        {
            DryadLogger.LogInformation("ProcessYarnUpdate", "Task {0} on node {1} is in state {2}", v.Id, v.Node,
                                       v.State);
            VertexChangeEventArgs e = new VertexChangeEventArgs(v.Id);

            e.NewNode = v.Node;
            e.NewState = YarnTaskStateToVertexTaskState(v.State);
            e.NewRequeueCount = v.RequeueCount;

            if (m_vertices[v.Id] != null)
            {
                e.OldNode = m_vertices[v.Id].Node;
                e.OldState = YarnTaskStateToVertexTaskState(m_vertices[v.Id].State);
                e.OldRequeueCount = m_vertices[v.Id].RequeueCount;
            }

            if (e.NewRequeueCount != e.OldRequeueCount)
            {
                DryadLogger.LogInformation("ProcessYarnUpdate", "Task {0} requeue count changed from {1} to {2}",
                                           v.Id, e.OldRequeueCount, e.NewRequeueCount);
            }

            // Update current vertex state
            m_vertices[v.Id] = v;
            m_vertexChangeEvent(this, e);
            //m_taskChangeEvt.Set();
        }