/// <exception cref="System.IO.IOException"/>
        protected internal virtual void CheckRemoveLabelsFromNode(IDictionary <NodeId, ICollection
                                                                               <string> > removeLabelsFromNode)
        {
            // check all labels being added existed
            ICollection <string> knownLabels = labelCollections.Keys;

            foreach (KeyValuePair <NodeId, ICollection <string> > entry in removeLabelsFromNode)
            {
                NodeId nodeId = entry.Key;
                ICollection <string> labels = entry.Value;
                if (!knownLabels.ContainsAll(labels))
                {
                    string msg = "Not all labels being removed contained by known " + "label collections, please check"
                                 + ", removed labels=[" + StringUtils.Join(labels, ",") + "]";
                    Log.Error(msg);
                    throw new IOException(msg);
                }
                ICollection <string> originalLabels = null;
                bool nodeExisted = false;
                if (WildcardPort != nodeId.GetPort())
                {
                    CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId);
                    if (nm != null)
                    {
                        originalLabels = nm.labels;
                        nodeExisted    = true;
                    }
                }
                else
                {
                    CommonNodeLabelsManager.Host host = nodeCollections[nodeId.GetHost()];
                    if (null != host)
                    {
                        originalLabels = host.labels;
                        nodeExisted    = true;
                    }
                }
                if (!nodeExisted)
                {
                    string msg = "Try to remove labels from NM=" + nodeId + ", but the NM doesn't existed";
                    Log.Error(msg);
                    throw new IOException(msg);
                }
                // the labels will never be null
                if (labels.IsEmpty())
                {
                    continue;
                }
                // originalLabels may be null,
                // because when a Node is created, Node.labels can be null.
                if (originalLabels == null || !originalLabels.ContainsAll(labels))
                {
                    string msg = "Try to remove labels = [" + StringUtils.Join(labels, ",") + "], but not all labels contained by NM="
                                 + nodeId;
                    Log.Error(msg);
                    throw new IOException(msg);
                }
            }
        }
Пример #2
0
        public static Token NewContainerToken(NodeId nodeId, byte[] password, ContainerTokenIdentifier
                                              tokenIdentifier)
        {
            // RPC layer client expects ip:port as service for tokens
            IPEndPoint addr = NetUtils.CreateSocketAddrForHost(nodeId.GetHost(), nodeId.GetPort
                                                                   ());
            // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token
            Token containerToken = NewToken <Token>(tokenIdentifier.GetBytes(), ContainerTokenIdentifier
                                                    .Kind.ToString(), password, SecurityUtil.BuildTokenService(addr).ToString());

            return(containerToken);
        }
        public static Org.Apache.Hadoop.Yarn.Api.Records.Token NewInstance(byte[] password
                                                                           , NMTokenIdentifier identifier)
        {
            NodeId nodeId = identifier.GetNodeId();
            // RPC layer client expects ip:port as service for tokens
            IPEndPoint addr = NetUtils.CreateSocketAddrForHost(nodeId.GetHost(), nodeId.GetPort
                                                                   ());

            Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = Org.Apache.Hadoop.Yarn.Api.Records.Token
                                                               .NewInstance(identifier.GetBytes(), NMTokenIdentifier.Kind.ToString(), password,
                                                                            SecurityUtil.BuildTokenService(addr).ToString());
            return(nmToken);
        }
Пример #4
0
        protected internal virtual ContainerManagementProtocol GetContainerManagementProtocolProxy
            (YarnRPC rpc, Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken, NodeId nodeId, string
            user)
        {
            ContainerManagementProtocol proxy;
            UserGroupInformation        ugi = UserGroupInformation.CreateRemoteUser(user);
            IPEndPoint addr = NetUtils.CreateSocketAddr(nodeId.GetHost(), nodeId.GetPort());

            if (nmToken != null)
            {
                ugi.AddToken(ConverterUtils.ConvertFromYarn(nmToken, addr));
            }
            proxy = NMProxy.CreateNMProxy <ContainerManagementProtocol>(conf, ugi, rpc, addr);
            return(proxy);
        }
Пример #5
0
        public static FiCaSchedulerNode GetMockNode(string host, string rack, int port, int
                                                    capability)
        {
            NodeId nodeId = Org.Mockito.Mockito.Mock <NodeId>();

            Org.Mockito.Mockito.When(nodeId.GetHost()).ThenReturn(host);
            Org.Mockito.Mockito.When(nodeId.GetPort()).ThenReturn(port);
            RMNode rmNode = Org.Mockito.Mockito.Mock <RMNode>();

            Org.Mockito.Mockito.When(rmNode.GetNodeID()).ThenReturn(nodeId);
            Org.Mockito.Mockito.When(rmNode.GetTotalCapability()).ThenReturn(Resources.CreateResource
                                                                                 (capability, 1));
            Org.Mockito.Mockito.When(rmNode.GetNodeAddress()).ThenReturn(host + ":" + port);
            Org.Mockito.Mockito.When(rmNode.GetHostName()).ThenReturn(host);
            Org.Mockito.Mockito.When(rmNode.GetRackName()).ThenReturn(rack);
            FiCaSchedulerNode node = Org.Mockito.Mockito.Spy(new FiCaSchedulerNode(rmNode, false
                                                                                   ));

            Log.Info("node = " + host + " avail=" + node.GetAvailableResource());
            return(node);
        }
Пример #6
0
        public static ImmutableMap <string, ICollection <NodeId> > TransposeNodeToLabels(IDictionary
                                                                                         <NodeId, ICollection <string> > mapNodeToLabels)
        {
            IDictionary <string, ICollection <NodeId> > mapLabelsToNodes = new Dictionary <string
                                                                                           , ICollection <NodeId> >();

            foreach (KeyValuePair <NodeId, ICollection <string> > entry in mapNodeToLabels)
            {
                NodeId node = entry.Key;
                ICollection <string> setLabels = entry.Value;
                foreach (string label in setLabels)
                {
                    ICollection <NodeId> setNode = mapLabelsToNodes[label];
                    if (setNode == null)
                    {
                        setNode = new HashSet <NodeId>();
                    }
                    setNode.AddItem(NodeId.NewInstance(node.GetHost(), node.GetPort()));
                    mapLabelsToNodes[label] = setNode;
                }
            }
            return(ImmutableMap.CopyOf(mapLabelsToNodes));
        }
        /// <exception cref="System.IO.IOException"/>
        protected internal virtual void InternalUpdateLabelsOnNodes(IDictionary <NodeId, ICollection
                                                                                 <string> > nodeToLabels, CommonNodeLabelsManager.NodeLabelUpdateOperation op)
        {
            // do update labels from nodes
            IDictionary <NodeId, ICollection <string> > newNMToLabels = new Dictionary <NodeId, ICollection
                                                                                        <string> >();
            ICollection <string> oldLabels;

            foreach (KeyValuePair <NodeId, ICollection <string> > entry in nodeToLabels)
            {
                NodeId nodeId = entry.Key;
                ICollection <string> labels = entry.Value;
                CreateHostIfNonExisted(nodeId.GetHost());
                if (nodeId.GetPort() == WildcardPort)
                {
                    CommonNodeLabelsManager.Host host = nodeCollections[nodeId.GetHost()];
                    switch (op)
                    {
                    case CommonNodeLabelsManager.NodeLabelUpdateOperation.Remove:
                    {
                        RemoveNodeFromLabels(nodeId, labels);
                        host.labels.RemoveAll(labels);
                        foreach (CommonNodeLabelsManager.Node node in host.nms.Values)
                        {
                            if (node.labels != null)
                            {
                                node.labels.RemoveAll(labels);
                            }
                            RemoveNodeFromLabels(node.nodeId, labels);
                        }
                        break;
                    }

                    case CommonNodeLabelsManager.NodeLabelUpdateOperation.Add:
                    {
                        AddNodeToLabels(nodeId, labels);
                        Sharpen.Collections.AddAll(host.labels, labels);
                        foreach (CommonNodeLabelsManager.Node node_1 in host.nms.Values)
                        {
                            if (node_1.labels != null)
                            {
                                Sharpen.Collections.AddAll(node_1.labels, labels);
                            }
                            AddNodeToLabels(node_1.nodeId, labels);
                        }
                        break;
                    }

                    case CommonNodeLabelsManager.NodeLabelUpdateOperation.Replace:
                    {
                        ReplaceNodeForLabels(nodeId, host.labels, labels);
                        host.labels.Clear();
                        Sharpen.Collections.AddAll(host.labels, labels);
                        foreach (CommonNodeLabelsManager.Node node_2 in host.nms.Values)
                        {
                            ReplaceNodeForLabels(node_2.nodeId, node_2.labels, labels);
                            node_2.labels = null;
                        }
                        break;
                    }

                    default:
                    {
                        break;
                    }
                    }
                    newNMToLabels[nodeId] = host.labels;
                }
                else
                {
                    if (EnumSet.Of(CommonNodeLabelsManager.NodeLabelUpdateOperation.Add, CommonNodeLabelsManager.NodeLabelUpdateOperation
                                   .Replace).Contains(op))
                    {
                        // Add and replace
                        CreateNodeIfNonExisted(nodeId);
                        CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId);
                        switch (op)
                        {
                        case CommonNodeLabelsManager.NodeLabelUpdateOperation.Add:
                        {
                            AddNodeToLabels(nodeId, labels);
                            if (nm.labels == null)
                            {
                                nm.labels = new HashSet <string>();
                            }
                            Sharpen.Collections.AddAll(nm.labels, labels);
                            break;
                        }

                        case CommonNodeLabelsManager.NodeLabelUpdateOperation.Replace:
                        {
                            oldLabels = GetLabelsByNode(nodeId);
                            ReplaceNodeForLabels(nodeId, oldLabels, labels);
                            if (nm.labels == null)
                            {
                                nm.labels = new HashSet <string>();
                            }
                            nm.labels.Clear();
                            Sharpen.Collections.AddAll(nm.labels, labels);
                            break;
                        }

                        default:
                        {
                            break;
                        }
                        }
                        newNMToLabels[nodeId] = nm.labels;
                    }
                    else
                    {
                        // remove
                        RemoveNodeFromLabels(nodeId, labels);
                        CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId);
                        if (nm.labels != null)
                        {
                            nm.labels.RemoveAll(labels);
                            newNMToLabels[nodeId] = nm.labels;
                        }
                    }
                }
            }
            if (null != dispatcher)
            {
                dispatcher.GetEventHandler().Handle(new UpdateNodeToLabelsMappingsEvent(newNMToLabels
                                                                                        ));
            }
            // shows node->labels we added
            Log.Info(op.ToString() + " labels on nodes:");
            foreach (KeyValuePair <NodeId, ICollection <string> > entry_1 in newNMToLabels)
            {
                Log.Info("  NM=" + entry_1.Key + ", labels=[" + StringUtils.Join(entry_1.Value.GetEnumerator
                                                                                     (), ",") + "]");
            }
        }
Пример #8
0
        // Protected. For tests.
        protected internal virtual ContainerManagementProtocol GetContainerMgrProxy(ContainerId
                                                                                    containerId)
        {
            NodeId     node = masterContainer.GetNodeId();
            IPEndPoint containerManagerConnectAddress = NetUtils.CreateSocketAddrForHost(node
                                                                                         .GetHost(), node.GetPort());
            YarnRPC rpc = GetYarnRPC();
            UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(containerId
                                                                                     .GetApplicationAttemptId().ToString());
            string user = rmContext.GetRMApps()[containerId.GetApplicationAttemptId().GetApplicationId
                                                    ()].GetUser();
            Token token = rmContext.GetNMTokenSecretManager().CreateNMToken(containerId.GetApplicationAttemptId
                                                                                (), node, user);

            currentUser.AddToken(ConverterUtils.ConvertFromYarn(token, containerManagerConnectAddress
                                                                ));
            return(NMProxy.CreateNMProxy <ContainerManagementProtocol>(conf, currentUser, rpc,
                                                                       containerManagerConnectAddress));
        }
Пример #9
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        public virtual RegisterNodeManagerResponse RegisterNodeManager(RegisterNodeManagerRequest
                                                                       request)
        {
            NodeId   nodeId                      = request.GetNodeId();
            string   host                        = nodeId.GetHost();
            int      cmPort                      = nodeId.GetPort();
            int      httpPort                    = request.GetHttpPort();
            Resource capability                  = request.GetResource();
            string   nodeManagerVersion          = request.GetNMVersion();
            RegisterNodeManagerResponse response = recordFactory.NewRecordInstance <RegisterNodeManagerResponse
                                                                                    >();

            if (!minimumNodeManagerVersion.Equals("NONE"))
            {
                if (minimumNodeManagerVersion.Equals("EqualToRM"))
                {
                    minimumNodeManagerVersion = YarnVersionInfo.GetVersion();
                }
                if ((nodeManagerVersion == null) || (VersionUtil.CompareVersions(nodeManagerVersion
                                                                                 , minimumNodeManagerVersion)) < 0)
                {
                    string message = "Disallowed NodeManager Version " + nodeManagerVersion + ", is less than the minimum version "
                                     + minimumNodeManagerVersion + " sending SHUTDOWN signal to " + "NodeManager.";
                    Log.Info(message);
                    response.SetDiagnosticsMessage(message);
                    response.SetNodeAction(NodeAction.Shutdown);
                    return(response);
                }
            }
            // Check if this node is a 'valid' node
            if (!this.nodesListManager.IsValidNode(host))
            {
                string message = "Disallowed NodeManager from  " + host + ", Sending SHUTDOWN signal to the NodeManager.";
                Log.Info(message);
                response.SetDiagnosticsMessage(message);
                response.SetNodeAction(NodeAction.Shutdown);
                return(response);
            }
            // Check if this node has minimum allocations
            if (capability.GetMemory() < minAllocMb || capability.GetVirtualCores() < minAllocVcores)
            {
                string message = "NodeManager from  " + host + " doesn't satisfy minimum allocations, Sending SHUTDOWN"
                                 + " signal to the NodeManager.";
                Log.Info(message);
                response.SetDiagnosticsMessage(message);
                response.SetNodeAction(NodeAction.Shutdown);
                return(response);
            }
            response.SetContainerTokenMasterKey(containerTokenSecretManager.GetCurrentKey());
            response.SetNMTokenMasterKey(nmTokenSecretManager.GetCurrentKey());
            RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort, Resolve
                                               (host), capability, nodeManagerVersion);
            RMNode oldNode = this.rmContext.GetRMNodes().PutIfAbsent(nodeId, rmNode);

            if (oldNode == null)
            {
                this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeStartedEvent(nodeId
                                                                                               , request.GetNMContainerStatuses(), request.GetRunningApplications()));
            }
            else
            {
                Log.Info("Reconnect from the node at: " + host);
                this.nmLivelinessMonitor.Unregister(nodeId);
                // Reset heartbeat ID since node just restarted.
                oldNode.ResetLastNodeHeartBeatResponse();
                this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeReconnectEvent(
                                                                            nodeId, rmNode, request.GetRunningApplications(), request.GetNMContainerStatuses
                                                                                ()));
            }
            // On every node manager register we will be clearing NMToken keys if
            // present for any running application.
            this.nmTokenSecretManager.RemoveNodeKey(nodeId);
            this.nmLivelinessMonitor.Register(nodeId);
            // Handle received container status, this should be processed after new
            // RMNode inserted
            if (!rmContext.IsWorkPreservingRecoveryEnabled())
            {
                if (!request.GetNMContainerStatuses().IsEmpty())
                {
                    Log.Info("received container statuses on node manager register :" + request.GetNMContainerStatuses
                                 ());
                    foreach (NMContainerStatus status in request.GetNMContainerStatuses())
                    {
                        HandleNMContainerStatus(status, nodeId);
                    }
                }
            }
            string message_1 = "NodeManager from node " + host + "(cmPort: " + cmPort + " httpPort: "
                               + httpPort + ") " + "registered with capability: " + capability + ", assigned nodeId "
                               + nodeId;

            Log.Info(message_1);
            response.SetNodeAction(NodeAction.Normal);
            response.SetRMIdentifier(ResourceManager.GetClusterTimeStamp());
            response.SetRMVersion(YarnVersionInfo.GetVersion());
            return(response);
        }