Пример #1
0
        public static TaskTrackerInfo FromYarn(NodeReport node)
        {
            TaskTrackerInfo taskTracker = new TaskTrackerInfo(TtNamePrefix + node.GetNodeId()
                                                              .ToString());

            return(taskTracker);
        }
Пример #2
0
        public void Save(int id, PageAnalysis pageAnalysis)
        {
            if (id < 0)
            {
                throw new ArgumentException(nameof(id));
            }
            if (pageAnalysis == null)
            {
                throw new ArgumentNullException(nameof(pageAnalysis));
            }

            if (_nodeReportRepository.TableExists)
            {
                var scoreReport = _pageScoreSerializer.Serialize(pageAnalysis.Score);

                var nodeReport = new NodeReport
                {
                    Id           = id,
                    FocusKeyword = pageAnalysis.FocusKeyword,
                    Report       = scoreReport,
                    CreatedOn    = DateTime.Now,
                    UpdatedOn    = DateTime.Now
                };

                Save(nodeReport);
            }
        }
Пример #3
0
 private YarnProtos.NodeReportProto ConvertToProtoFormat(NodeReport t)
 {
     lock (this)
     {
         return(((NodeReportPBImpl)t).GetProto());
     }
 }
        public void Save(int id, string focusKeyword, PageAnalysis pageAnalysis)
        {
            if (_nodeReportRepository.DatabaseExists())
            {
                var serializer = new JavaScriptSerializer();

                var json = serializer.Serialize(pageAnalysis.Score);

                var nodeReport = _nodeReportRepository.GetById(id);
                if (nodeReport == null)
                {
                    nodeReport = new NodeReport
                    {
                        Id           = id,
                        FocusKeyword = focusKeyword,
                        Report       = json
                    };

                    _nodeReportRepository.Insert(nodeReport);
                }
                else
                {
                    nodeReport.FocusKeyword = focusKeyword;
                    nodeReport.Report       = json;

                    _nodeReportRepository.Update(nodeReport);
                }
            }
        }
Пример #5
0
        /// <summary>Prints the node report for node id.</summary>
        /// <param name="nodeIdStr"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        private void PrintNodeStatus(string nodeIdStr)
        {
            NodeId             nodeId      = ConverterUtils.ToNodeId(nodeIdStr);
            IList <NodeReport> nodesReport = client.GetNodeReports();
            // Use PrintWriter.println, which uses correct platform line ending.
            ByteArrayOutputStream baos          = new ByteArrayOutputStream();
            PrintWriter           nodeReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding
                                                                                             ("UTF-8")));
            NodeReport nodeReport = null;

            foreach (NodeReport report in nodesReport)
            {
                if (!report.GetNodeId().Equals(nodeId))
                {
                    continue;
                }
                nodeReport = report;
                nodeReportStr.WriteLine("Node Report : ");
                nodeReportStr.Write("\tNode-Id : ");
                nodeReportStr.WriteLine(nodeReport.GetNodeId());
                nodeReportStr.Write("\tRack : ");
                nodeReportStr.WriteLine(nodeReport.GetRackName());
                nodeReportStr.Write("\tNode-State : ");
                nodeReportStr.WriteLine(nodeReport.GetNodeState());
                nodeReportStr.Write("\tNode-Http-Address : ");
                nodeReportStr.WriteLine(nodeReport.GetHttpAddress());
                nodeReportStr.Write("\tLast-Health-Update : ");
                nodeReportStr.WriteLine(DateFormatUtils.Format(Sharpen.Extensions.CreateDate(nodeReport
                                                                                             .GetLastHealthReportTime()), "E dd/MMM/yy hh:mm:ss:SSzz"));
                nodeReportStr.Write("\tHealth-Report : ");
                nodeReportStr.WriteLine(nodeReport.GetHealthReport());
                nodeReportStr.Write("\tContainers : ");
                nodeReportStr.WriteLine(nodeReport.GetNumContainers());
                nodeReportStr.Write("\tMemory-Used : ");
                nodeReportStr.WriteLine((nodeReport.GetUsed() == null) ? "0MB" : (nodeReport.GetUsed
                                                                                      ().GetMemory() + "MB"));
                nodeReportStr.Write("\tMemory-Capacity : ");
                nodeReportStr.WriteLine(nodeReport.GetCapability().GetMemory() + "MB");
                nodeReportStr.Write("\tCPU-Used : ");
                nodeReportStr.WriteLine((nodeReport.GetUsed() == null) ? "0 vcores" : (nodeReport
                                                                                       .GetUsed().GetVirtualCores() + " vcores"));
                nodeReportStr.Write("\tCPU-Capacity : ");
                nodeReportStr.WriteLine(nodeReport.GetCapability().GetVirtualCores() + " vcores");
                nodeReportStr.Write("\tNode-Labels : ");
                // Create a List for node labels since we need it get sorted
                IList <string> nodeLabelsList = new AList <string>(report.GetNodeLabels());
                nodeLabelsList.Sort();
                nodeReportStr.WriteLine(StringUtils.Join(nodeLabelsList.GetEnumerator(), ','));
            }
            if (nodeReport == null)
            {
                nodeReportStr.Write("Could not find the node report for node id : " + nodeIdStr);
            }
            nodeReportStr.Close();
            sysout.WriteLine(baos.ToString("UTF-8"));
        }
Пример #6
0
            public virtual IList <NodeReport> CreateFakeNodeReports()
            {
                NodeId     nodeId = NodeId.NewInstance("localhost", 0);
                NodeReport report = NodeReport.NewInstance(nodeId, NodeState.Running, "localhost"
                                                           , "rack1", null, null, 4, null, 1000l, null);
                IList <NodeReport> reports = new AList <NodeReport>();

                reports.AddItem(report);
                return(reports);
            }
Пример #7
0
        void ReceiveReport(NodeReport r)
        {
            var state = nodeStates[r.Node.NodeID];

            state.RecordContact();

            if (!state.Muted)
            {
                Console.WriteLine($"{DateTimeOffset.Now:t} Report from node {r.Node}: {r.GetType().Name} [{r}]");
            }

            SaveState();
        }
Пример #8
0
        public void Save(int id, PageAnalysis pageAnalysis)
        {
            var scoreReport = _javaScriptSerializer.Serialize(pageAnalysis.Score);

            var nodeReport = new NodeReport
            {
                Id           = id,
                FocusKeyword = pageAnalysis.FocusKeyword,
                Report       = scoreReport,
                CreatedOn    = DateTime.Now,
                UpdatedOn    = DateTime.Now
            };

            CreateOrUpdateNodeReport(nodeReport);
        }
Пример #9
0
        private void CreateOrUpdateNodeReport(NodeReport nodeReport)
        {
            var dbNodeReport = _nodeReportRepository.GetById(nodeReport.Id);

            if (dbNodeReport == null)
            {
                _nodeReportRepository.Insert(nodeReport);
            }
            else
            {
                dbNodeReport.FocusKeyword = nodeReport.FocusKeyword;
                dbNodeReport.Report       = nodeReport.Report;
                dbNodeReport.UpdatedOn    = nodeReport.UpdatedOn;

                _nodeReportRepository.Update(nodeReport);
            }
        }
Пример #10
0
        public static NodeReport NewNodeReport(NodeId nodeId, NodeState nodeState, string
                                               httpAddress, string rackName, Resource used, Resource capability, int numContainers
                                               , string healthReport, long lastHealthReportTime, ICollection <string> nodeLabels
                                               )
        {
            NodeReport nodeReport = recordFactory.NewRecordInstance <NodeReport>();

            nodeReport.SetNodeId(nodeId);
            nodeReport.SetNodeState(nodeState);
            nodeReport.SetHttpAddress(httpAddress);
            nodeReport.SetRackName(rackName);
            nodeReport.SetUsed(used);
            nodeReport.SetCapability(capability);
            nodeReport.SetNumContainers(numContainers);
            nodeReport.SetHealthReport(healthReport);
            nodeReport.SetLastHealthReportTime(lastHealthReportTime);
            nodeReport.SetNodeLabels(nodeLabels);
            return(nodeReport);
        }
Пример #11
0
        public NodeAnalysisModel(NodeReport nodeReport)
        {
            _nodeReport     = nodeReport;
            Label           = "Node: " + NodeId;
            TotalKeyCount   = nodeReport.KeyCount;
            TotalChildCount = nodeReport.ChildNodeCount;

            foreach (var rr in _nodeReport.RelatedResourceLists)
            {
                var btr = new BTreeAnalysisModel(rr);
                Children.Add(btr);
            }

            foreach (var nr in _nodeReport.Children)
            {
                var nam = new NodeAnalysisModel(nr);
                Children.Add(nam);
                TotalChildCount += nam.TotalChildCount;
                TotalKeyCount   += nam.TotalKeyCount;
            }

            AvgKeysPerNode = (double)TotalKeyCount / (TotalChildCount + 1);
        }
Пример #12
0
        public virtual void TestUpdatedNodes()
        {
            int   runCount = 0;
            MRApp app      = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName
                                                            , true, ++runCount);
            Configuration conf = new Configuration();

            // after half of the map completion, reduce will start
            conf.SetFloat(MRJobConfig.CompletedMapsForReduceSlowstart, 0.5f);
            // uberization forces full slowstart (1.0), so disable that
            conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
            app.WaitForState(job, JobState.Running);
            NUnit.Framework.Assert.AreEqual("Num tasks not correct", 4, job.GetTasks().Count);
            IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
            Task mapTask1         = it.Next();
            Task mapTask2         = it.Next();

            // all maps must be running
            app.WaitForState(mapTask1, TaskState.Running);
            app.WaitForState(mapTask2, TaskState.Running);
            TaskAttempt task1Attempt = mapTask1.GetAttempts().Values.GetEnumerator().Next();
            TaskAttempt task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next();
            NodeId      node1        = task1Attempt.GetNodeId();
            NodeId      node2        = task2Attempt.GetNodeId();

            NUnit.Framework.Assert.AreEqual(node1, node2);
            // send the done signal to the task
            app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID
                                                                               (), TaskAttemptEventType.TaDone));
            app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID
                                                                               (), TaskAttemptEventType.TaDone));
            // all maps must be succeeded
            app.WaitForState(mapTask1, TaskState.Succeeded);
            app.WaitForState(mapTask2, TaskState.Succeeded);
            TaskAttemptCompletionEvent[] events = job.GetTaskAttemptCompletionEvents(0, 100);
            NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for success", 2, events
                                            .Length);
            // send updated nodes info
            AList <NodeReport> updatedNodes = new AList <NodeReport>();
            NodeReport         nr           = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <NodeReport
                                                                                                              >();

            nr.SetNodeId(node1);
            nr.SetNodeState(NodeState.Unhealthy);
            updatedNodes.AddItem(nr);
            app.GetContext().GetEventHandler().Handle(new JobUpdatedNodesEvent(job.GetID(), updatedNodes
                                                                               ));
            app.WaitForState(task1Attempt, TaskAttemptState.Killed);
            app.WaitForState(task2Attempt, TaskAttemptState.Killed);
            events = job.GetTaskAttemptCompletionEvents(0, 100);
            NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for killed",
                                            4, events.Length);
            // all maps must be back to running
            app.WaitForState(mapTask1, TaskState.Running);
            app.WaitForState(mapTask2, TaskState.Running);
            IEnumerator <TaskAttempt> itr = mapTask1.GetAttempts().Values.GetEnumerator();

            itr.Next();
            task1Attempt = itr.Next();
            // send the done signal to the task
            app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID
                                                                               (), TaskAttemptEventType.TaDone));
            // map1 must be succeeded. map2 must be running
            app.WaitForState(mapTask1, TaskState.Succeeded);
            app.WaitForState(mapTask2, TaskState.Running);
            events = job.GetTaskAttemptCompletionEvents(0, 100);
            NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success",
                                            5, events.Length);
            // Crash the app again.
            app.Stop();
            // rerun
            // in rerun the 1st map will be recovered from previous run
            app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName,
                                                 false, ++runCount);
            conf = new Configuration();
            conf.SetBoolean(MRJobConfig.MrAmJobRecoveryEnable, true);
            conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
            job = app.Submit(conf);
            app.WaitForState(job, JobState.Running);
            NUnit.Framework.Assert.AreEqual("No of tasks not correct", 4, job.GetTasks().Count
                                            );
            it       = job.GetTasks().Values.GetEnumerator();
            mapTask1 = it.Next();
            mapTask2 = it.Next();
            Task reduceTask1 = it.Next();
            Task reduceTask2 = it.Next();

            // map 1 will be recovered, no need to send done
            app.WaitForState(mapTask1, TaskState.Succeeded);
            app.WaitForState(mapTask2, TaskState.Running);
            events = job.GetTaskAttemptCompletionEvents(0, 100);
            NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for killed & success of map1"
                                            , 2, events.Length);
            task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next();
            app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID
                                                                               (), TaskAttemptEventType.TaDone));
            app.WaitForState(mapTask2, TaskState.Succeeded);
            events = job.GetTaskAttemptCompletionEvents(0, 100);
            NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success",
                                            3, events.Length);
            app.WaitForState(reduceTask1, TaskState.Running);
            app.WaitForState(reduceTask2, TaskState.Running);
            TaskAttempt task3Attempt = reduceTask1.GetAttempts().Values.GetEnumerator().Next(
                );

            app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID
                                                                               (), TaskAttemptEventType.TaDone));
            app.WaitForState(reduceTask1, TaskState.Succeeded);
            app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID
                                                                               (), TaskAttemptEventType.TaKill));
            app.WaitForState(reduceTask1, TaskState.Succeeded);
            TaskAttempt task4Attempt = reduceTask2.GetAttempts().Values.GetEnumerator().Next(
                );

            app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task4Attempt.GetID
                                                                               (), TaskAttemptEventType.TaDone));
            app.WaitForState(reduceTask2, TaskState.Succeeded);
            events = job.GetTaskAttemptCompletionEvents(0, 100);
            NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for reduce success"
                                            , 5, events.Length);
            // job succeeds
            app.WaitForState(job, JobState.Succeeded);
        }
Пример #13
0
 public static long TotalNodeCount(this NodeReport root)
 {
     return(root.Children.Sum(c => c.TotalNodeCount()) + 1);
 }
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        public virtual AllocateResponse Allocate(AllocateRequest request)
        {
            AMRMTokenIdentifier  amrmTokenIdentifier = AuthorizeRequest();
            ApplicationAttemptId appAttemptId        = amrmTokenIdentifier.GetApplicationAttemptId();
            ApplicationId        applicationId       = appAttemptId.GetApplicationId();

            this.amLivelinessMonitor.ReceivedPing(appAttemptId);
            /* check if its in cache */
            ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId];
            if (Lock == null)
            {
                string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache.";
                Log.Error(message);
                throw new ApplicationAttemptNotFoundException(message);
            }
            lock (Lock)
            {
                AllocateResponse lastResponse = Lock.GetAllocateResponse();
                if (!HasApplicationMasterRegistered(appAttemptId))
                {
                    string message = "AM is not registered for known application attempt: " + appAttemptId
                                     + " or RM had restarted after AM registered . AM should re-register.";
                    Log.Info(message);
                    RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId
                                                                            ()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService"
                                             , message, applicationId, appAttemptId);
                    throw new ApplicationMasterNotRegisteredException(message);
                }
                if ((request.GetResponseId() + 1) == lastResponse.GetResponseId())
                {
                    /* old heartbeat */
                    return(lastResponse);
                }
                else
                {
                    if (request.GetResponseId() + 1 < lastResponse.GetResponseId())
                    {
                        string message = "Invalid responseId in AllocateRequest from application attempt: "
                                         + appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() +
                                                                                          1);
                        throw new InvalidApplicationMasterRequestException(message);
                    }
                }
                //filter illegal progress values
                float filteredProgress = request.GetProgress();
                if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity ||
                    filteredProgress < 0)
                {
                    request.SetProgress(0);
                }
                else
                {
                    if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity)
                    {
                        request.SetProgress(1);
                    }
                }
                // Send the status update to the appAttempt.
                this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent
                                                                            (appAttemptId, request.GetProgress()));
                IList <ResourceRequest>  ask                = request.GetAskList();
                IList <ContainerId>      release            = request.GetReleaseList();
                ResourceBlacklistRequest blacklistRequest   = request.GetResourceBlacklistRequest();
                IList <string>           blacklistAdditions = (blacklistRequest != null) ? blacklistRequest.
                                                              GetBlacklistAdditions() : Sharpen.Collections.EmptyList;
                IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals
                                                       () : Sharpen.Collections.EmptyList;
                RMApp app = this.rmContext.GetRMApps()[applicationId];
                // set label expression for Resource Requests if resourceName=ANY
                ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext();
                foreach (ResourceRequest req in ask)
                {
                    if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName
                                                                                               ()))
                    {
                        req.SetNodeLabelExpression(asc.GetNodeLabelExpression());
                    }
                }
                // sanity check
                try
                {
                    RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability
                                                                   (), app.GetQueue(), rScheduler, rmContext);
                }
                catch (InvalidResourceRequestException e)
                {
                    Log.Warn("Invalid resource ask by application " + appAttemptId, e);
                    throw;
                }
                try
                {
                    RMServerUtils.ValidateBlacklistRequest(blacklistRequest);
                }
                catch (InvalidResourceBlacklistRequestException e)
                {
                    Log.Warn("Invalid blacklist request by application " + appAttemptId, e);
                    throw;
                }
                // In the case of work-preserving AM restart, it's possible for the
                // AM to release containers from the earlier attempt.
                if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts
                        ())
                {
                    try
                    {
                        RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId);
                    }
                    catch (InvalidContainerReleaseException e)
                    {
                        Log.Warn("Invalid container release by application " + appAttemptId, e);
                        throw;
                    }
                }
                // Send new requests to appAttempt.
                Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions
                                                                 , blacklistRemovals);
                if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty())
                {
                    Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions
                             + ", " + "blacklistRemovals: " + blacklistRemovals);
                }
                RMAppAttempt     appAttempt       = app.GetRMAppAttempt(appAttemptId);
                AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse
                                                                                     >();
                if (!allocation.GetContainers().IsEmpty())
                {
                    allocateResponse.SetNMTokens(allocation.GetNMTokens());
                }
                // update the response with the deltas of node status changes
                IList <RMNode> updatedNodes = new AList <RMNode>();
                if (app.PullRMNodeUpdates(updatedNodes) > 0)
                {
                    IList <NodeReport> updatedNodeReports = new AList <NodeReport>();
                    foreach (RMNode rmNode in updatedNodes)
                    {
                        SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID
                                                                                               ());
                        Resource used          = BuilderUtils.NewResource(0, 0);
                        int      numContainers = 0;
                        if (schedulerNodeReport != null)
                        {
                            used          = schedulerNodeReport.GetUsedResource();
                            numContainers = schedulerNodeReport.GetNumContainers();
                        }
                        NodeId     nodeId = rmNode.GetNodeID();
                        NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode.
                                                                       GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers
                                                                       , rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels
                                                                           ());
                        updatedNodeReports.AddItem(report);
                    }
                    allocateResponse.SetUpdatedNodes(updatedNodeReports);
                }
                allocateResponse.SetAllocatedContainers(allocation.GetContainers());
                allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers
                                                                    ());
                allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1);
                allocateResponse.SetAvailableResources(allocation.GetResourceLimit());
                allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes());
                // add preemption to the allocateResponse message (if any)
                allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation));
                // update AMRMToken if the token is rolled-up
                MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData
                                                  ();
                if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier
                    .GetKeyId())
                {
                    RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt;
                    Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt
                                                                                             .GetAMRMToken();
                    if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId())
                    {
                        Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: "
                                 + applicationId);
                        amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId
                                                                                                );
                        appAttemptImpl.SetAMRMToken(amrmToken);
                    }
                    allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance
                                                      (amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword
                                                          (), amrmToken.GetService().ToString()));
                }

                /*
                 * As we are updating the response inside the lock object so we don't
                 * need to worry about unregister call occurring in between (which
                 * removes the lock object).
                 */
                Lock.SetAllocateResponse(allocateResponse);
                return(allocateResponse);
            }
        }
Пример #15
0
        public virtual void TestAMRMUnusableNodes()
        {
            MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 10000);
            MockNM nm2 = rm.RegisterNode("127.0.0.2:1234", 10000);
            MockNM nm3 = rm.RegisterNode("127.0.0.3:1234", 10000);
            MockNM nm4 = rm.RegisterNode("127.0.0.4:1234", 10000);

            dispatcher.Await();
            RMApp app1 = rm.SubmitApp(2000);

            // Trigger the scheduling so the AM gets 'launched' on nm1
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
            MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());

            // register AM returns no unusable node
            am1.RegisterAppAttempt();
            // allocate request returns no updated node
            AllocateRequest allocateRequest1 = AllocateRequest.NewInstance(0, 0F, null, null,
                                                                           null);
            AllocateResponse response1 = Allocate(attempt1.GetAppAttemptId(), allocateRequest1
                                                  );
            IList <NodeReport> updatedNodes = response1.GetUpdatedNodes();

            NUnit.Framework.Assert.AreEqual(0, updatedNodes.Count);
            SyncNodeHeartbeat(nm4, false);
            // allocate request returns updated node
            allocateRequest1 = AllocateRequest.NewInstance(response1.GetResponseId(), 0F, null
                                                           , null, null);
            response1    = Allocate(attempt1.GetAppAttemptId(), allocateRequest1);
            updatedNodes = response1.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count);
            NodeReport nr = updatedNodes.GetEnumerator().Next();

            NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId());
            NUnit.Framework.Assert.AreEqual(NodeState.Unhealthy, nr.GetNodeState());
            // resending the allocate request returns the same result
            response1    = Allocate(attempt1.GetAppAttemptId(), allocateRequest1);
            updatedNodes = response1.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count);
            nr = updatedNodes.GetEnumerator().Next();
            NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId());
            NUnit.Framework.Assert.AreEqual(NodeState.Unhealthy, nr.GetNodeState());
            SyncNodeLost(nm3);
            // subsequent allocate request returns delta
            allocateRequest1 = AllocateRequest.NewInstance(response1.GetResponseId(), 0F, null
                                                           , null, null);
            response1    = Allocate(attempt1.GetAppAttemptId(), allocateRequest1);
            updatedNodes = response1.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count);
            nr = updatedNodes.GetEnumerator().Next();
            NUnit.Framework.Assert.AreEqual(nm3.GetNodeId(), nr.GetNodeId());
            NUnit.Framework.Assert.AreEqual(NodeState.Lost, nr.GetNodeState());
            // registering another AM gives it the complete failed list
            RMApp app2 = rm.SubmitApp(2000);

            // Trigger nm2 heartbeat so that AM gets launched on it
            nm2.NodeHeartbeat(true);
            RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
            MockAM       am2      = rm.SendAMLaunched(attempt2.GetAppAttemptId());

            // register AM returns all unusable nodes
            am2.RegisterAppAttempt();
            // allocate request returns no updated node
            AllocateRequest allocateRequest2 = AllocateRequest.NewInstance(0, 0F, null, null,
                                                                           null);
            AllocateResponse response2 = Allocate(attempt2.GetAppAttemptId(), allocateRequest2
                                                  );

            updatedNodes = response2.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(0, updatedNodes.Count);
            SyncNodeHeartbeat(nm4, true);
            // both AM's should get delta updated nodes
            allocateRequest1 = AllocateRequest.NewInstance(response1.GetResponseId(), 0F, null
                                                           , null, null);
            response1    = Allocate(attempt1.GetAppAttemptId(), allocateRequest1);
            updatedNodes = response1.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count);
            nr = updatedNodes.GetEnumerator().Next();
            NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId());
            NUnit.Framework.Assert.AreEqual(NodeState.Running, nr.GetNodeState());
            allocateRequest2 = AllocateRequest.NewInstance(response2.GetResponseId(), 0F, null
                                                           , null, null);
            response2    = Allocate(attempt2.GetAppAttemptId(), allocateRequest2);
            updatedNodes = response2.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count);
            nr = updatedNodes.GetEnumerator().Next();
            NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId());
            NUnit.Framework.Assert.AreEqual(NodeState.Running, nr.GetNodeState());
            // subsequent allocate calls should return no updated nodes
            allocateRequest2 = AllocateRequest.NewInstance(response2.GetResponseId(), 0F, null
                                                           , null, null);
            response2    = Allocate(attempt2.GetAppAttemptId(), allocateRequest2);
            updatedNodes = response2.GetUpdatedNodes();
            NUnit.Framework.Assert.AreEqual(0, updatedNodes.Count);
        }
Пример #16
0
        /// <summary>
        /// 绑定报表数据
        /// </summary>
        /// <returns></returns>
        public override ArrayList BindingSource()
        {
            if (this.ReportStencil.Count > 0)
            {
                this.LoadReportSource();
                this.ReportResult = new ArrayList();
                NodeReport newxm = new NodeReport();
                int        i     = 1;
                newxm.ID         = i++;
                newxm.PID        = -1;
                newxm.ReportName = this.StructSource.ModelProject.Rows[0]["Name"].ToString();
                this.ReportResult.Add(newxm);
                _ObjectReport[] m_ObjectReport = this.ReportStencil.Cast <_ObjectReport>().Where(p => p.ReportType == "项目报表").ToArray();
                foreach (_ObjectReport item in m_ObjectReport)
                {
                    item.ID   = i++;
                    item.PID  = 1;
                    item.XMMC = this.StructSource.ModelProject.Rows[0]["Name"].ToString();
                    DataTable dt = this.ReportSource.Tables[item.Method];
                    if (dt != null)
                    {
                        item.DataSource = dt;
                    }
                    this.ReportResult.Add(item);
                }

                DataRow[]          drs = this.StructSource.ModelProject.Select("DEEP=2", "ID");
                _UnitProjectReport m_UnitProjectReport = null;
                foreach (DataRow item in drs)
                {
                    _UnitProject info = item["UnitProject"] as _UnitProject;
                    if (info == null)
                    {
                        info = new _UnitProject();
                        _ObjectSource.GetObject(info, item);
                        this.CurrBusiness.Load(info);
                    }
                    m_UnitProjectReport               = new _UnitProjectReport();
                    m_UnitProjectReport.UnID          = info.ActiviteId != 0 ? info.ActiviteId : info.ID;//Convert.ToInt32(item["ID"]);
                    m_UnitProjectReport.StructSource  = info.StructSource;
                    m_UnitProjectReport.ReportStencil = this.ReportStencil.Copy() as _List;
                    m_UnitProjectReport.BindingSource();
                    if (m_UnitProjectReport.ReportResult != null)
                    {
                        _ObjectReport m_dw = m_UnitProjectReport.ReportResult.Cast <_ObjectReport>().Where(p => p is NodeReport).FirstOrDefault();
                        if (m_dw != null)
                        {
                            m_dw.ID  = i++;
                            m_dw.PID = 1;
                            this.ReportResult.Add(m_dw);
                            foreach (_ObjectReport item_dw in m_UnitProjectReport.ReportResult)
                            {
                                this.TKBB_UnitProject(item_dw);
                                if (!(item_dw is NodeReport))
                                {
                                    item_dw.ID  = i++;
                                    item_dw.PID = m_dw.ID;
                                    this.ReportResult.Add(item_dw);
                                }
                            }
                        }
                    }
                }
            }
            return(this.ReportResult);
        }