/// <exception cref="System.IO.IOException"/> protected internal virtual void CheckRemoveLabelsFromNode(IDictionary <NodeId, ICollection <string> > removeLabelsFromNode) { // check all labels being added existed ICollection <string> knownLabels = labelCollections.Keys; foreach (KeyValuePair <NodeId, ICollection <string> > entry in removeLabelsFromNode) { NodeId nodeId = entry.Key; ICollection <string> labels = entry.Value; if (!knownLabels.ContainsAll(labels)) { string msg = "Not all labels being removed contained by known " + "label collections, please check" + ", removed labels=[" + StringUtils.Join(labels, ",") + "]"; Log.Error(msg); throw new IOException(msg); } ICollection <string> originalLabels = null; bool nodeExisted = false; if (WildcardPort != nodeId.GetPort()) { CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId); if (nm != null) { originalLabels = nm.labels; nodeExisted = true; } } else { CommonNodeLabelsManager.Host host = nodeCollections[nodeId.GetHost()]; if (null != host) { originalLabels = host.labels; nodeExisted = true; } } if (!nodeExisted) { string msg = "Try to remove labels from NM=" + nodeId + ", but the NM doesn't existed"; Log.Error(msg); throw new IOException(msg); } // the labels will never be null if (labels.IsEmpty()) { continue; } // originalLabels may be null, // because when a Node is created, Node.labels can be null. if (originalLabels == null || !originalLabels.ContainsAll(labels)) { string msg = "Try to remove labels = [" + StringUtils.Join(labels, ",") + "], but not all labels contained by NM=" + nodeId; Log.Error(msg); throw new IOException(msg); } } }
public static Token NewContainerToken(NodeId nodeId, byte[] password, ContainerTokenIdentifier tokenIdentifier) { // RPC layer client expects ip:port as service for tokens IPEndPoint addr = NetUtils.CreateSocketAddrForHost(nodeId.GetHost(), nodeId.GetPort ()); // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token Token containerToken = NewToken <Token>(tokenIdentifier.GetBytes(), ContainerTokenIdentifier .Kind.ToString(), password, SecurityUtil.BuildTokenService(addr).ToString()); return(containerToken); }
public static Org.Apache.Hadoop.Yarn.Api.Records.Token NewInstance(byte[] password , NMTokenIdentifier identifier) { NodeId nodeId = identifier.GetNodeId(); // RPC layer client expects ip:port as service for tokens IPEndPoint addr = NetUtils.CreateSocketAddrForHost(nodeId.GetHost(), nodeId.GetPort ()); Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = Org.Apache.Hadoop.Yarn.Api.Records.Token .NewInstance(identifier.GetBytes(), NMTokenIdentifier.Kind.ToString(), password, SecurityUtil.BuildTokenService(addr).ToString()); return(nmToken); }
protected internal virtual ContainerManagementProtocol GetContainerManagementProtocolProxy (YarnRPC rpc, Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken, NodeId nodeId, string user) { ContainerManagementProtocol proxy; UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user); IPEndPoint addr = NetUtils.CreateSocketAddr(nodeId.GetHost(), nodeId.GetPort()); if (nmToken != null) { ugi.AddToken(ConverterUtils.ConvertFromYarn(nmToken, addr)); } proxy = NMProxy.CreateNMProxy <ContainerManagementProtocol>(conf, ugi, rpc, addr); return(proxy); }
public static FiCaSchedulerNode GetMockNode(string host, string rack, int port, int capability) { NodeId nodeId = Org.Mockito.Mockito.Mock <NodeId>(); Org.Mockito.Mockito.When(nodeId.GetHost()).ThenReturn(host); Org.Mockito.Mockito.When(nodeId.GetPort()).ThenReturn(port); RMNode rmNode = Org.Mockito.Mockito.Mock <RMNode>(); Org.Mockito.Mockito.When(rmNode.GetNodeID()).ThenReturn(nodeId); Org.Mockito.Mockito.When(rmNode.GetTotalCapability()).ThenReturn(Resources.CreateResource (capability, 1)); Org.Mockito.Mockito.When(rmNode.GetNodeAddress()).ThenReturn(host + ":" + port); Org.Mockito.Mockito.When(rmNode.GetHostName()).ThenReturn(host); Org.Mockito.Mockito.When(rmNode.GetRackName()).ThenReturn(rack); FiCaSchedulerNode node = Org.Mockito.Mockito.Spy(new FiCaSchedulerNode(rmNode, false )); Log.Info("node = " + host + " avail=" + node.GetAvailableResource()); return(node); }
public static ImmutableMap <string, ICollection <NodeId> > TransposeNodeToLabels(IDictionary <NodeId, ICollection <string> > mapNodeToLabels) { IDictionary <string, ICollection <NodeId> > mapLabelsToNodes = new Dictionary <string , ICollection <NodeId> >(); foreach (KeyValuePair <NodeId, ICollection <string> > entry in mapNodeToLabels) { NodeId node = entry.Key; ICollection <string> setLabels = entry.Value; foreach (string label in setLabels) { ICollection <NodeId> setNode = mapLabelsToNodes[label]; if (setNode == null) { setNode = new HashSet <NodeId>(); } setNode.AddItem(NodeId.NewInstance(node.GetHost(), node.GetPort())); mapLabelsToNodes[label] = setNode; } } return(ImmutableMap.CopyOf(mapLabelsToNodes)); }
/// <exception cref="System.IO.IOException"/> protected internal virtual void InternalUpdateLabelsOnNodes(IDictionary <NodeId, ICollection <string> > nodeToLabels, CommonNodeLabelsManager.NodeLabelUpdateOperation op) { // do update labels from nodes IDictionary <NodeId, ICollection <string> > newNMToLabels = new Dictionary <NodeId, ICollection <string> >(); ICollection <string> oldLabels; foreach (KeyValuePair <NodeId, ICollection <string> > entry in nodeToLabels) { NodeId nodeId = entry.Key; ICollection <string> labels = entry.Value; CreateHostIfNonExisted(nodeId.GetHost()); if (nodeId.GetPort() == WildcardPort) { CommonNodeLabelsManager.Host host = nodeCollections[nodeId.GetHost()]; switch (op) { case CommonNodeLabelsManager.NodeLabelUpdateOperation.Remove: { RemoveNodeFromLabels(nodeId, labels); host.labels.RemoveAll(labels); foreach (CommonNodeLabelsManager.Node node in host.nms.Values) { if (node.labels != null) { node.labels.RemoveAll(labels); } RemoveNodeFromLabels(node.nodeId, labels); } break; } case CommonNodeLabelsManager.NodeLabelUpdateOperation.Add: { AddNodeToLabels(nodeId, labels); Sharpen.Collections.AddAll(host.labels, labels); foreach (CommonNodeLabelsManager.Node node_1 in host.nms.Values) { if (node_1.labels != null) { Sharpen.Collections.AddAll(node_1.labels, labels); } AddNodeToLabels(node_1.nodeId, labels); } break; } case CommonNodeLabelsManager.NodeLabelUpdateOperation.Replace: { ReplaceNodeForLabels(nodeId, host.labels, labels); host.labels.Clear(); Sharpen.Collections.AddAll(host.labels, labels); foreach (CommonNodeLabelsManager.Node node_2 in host.nms.Values) { ReplaceNodeForLabels(node_2.nodeId, node_2.labels, labels); node_2.labels = null; } break; } default: { break; } } newNMToLabels[nodeId] = host.labels; } else { if (EnumSet.Of(CommonNodeLabelsManager.NodeLabelUpdateOperation.Add, CommonNodeLabelsManager.NodeLabelUpdateOperation .Replace).Contains(op)) { // Add and replace CreateNodeIfNonExisted(nodeId); CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId); switch (op) { case CommonNodeLabelsManager.NodeLabelUpdateOperation.Add: { AddNodeToLabels(nodeId, labels); if (nm.labels == null) { nm.labels = new HashSet <string>(); } Sharpen.Collections.AddAll(nm.labels, labels); break; } case CommonNodeLabelsManager.NodeLabelUpdateOperation.Replace: { oldLabels = GetLabelsByNode(nodeId); ReplaceNodeForLabels(nodeId, oldLabels, labels); if (nm.labels == null) { nm.labels = new HashSet <string>(); } nm.labels.Clear(); Sharpen.Collections.AddAll(nm.labels, labels); break; } default: { break; } } newNMToLabels[nodeId] = nm.labels; } else { // remove RemoveNodeFromLabels(nodeId, labels); CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId); if (nm.labels != null) { nm.labels.RemoveAll(labels); newNMToLabels[nodeId] = nm.labels; } } } } if (null != dispatcher) { dispatcher.GetEventHandler().Handle(new UpdateNodeToLabelsMappingsEvent(newNMToLabels )); } // shows node->labels we added Log.Info(op.ToString() + " labels on nodes:"); foreach (KeyValuePair <NodeId, ICollection <string> > entry_1 in newNMToLabels) { Log.Info(" NM=" + entry_1.Key + ", labels=[" + StringUtils.Join(entry_1.Value.GetEnumerator (), ",") + "]"); } }
// Protected. For tests. protected internal virtual ContainerManagementProtocol GetContainerMgrProxy(ContainerId containerId) { NodeId node = masterContainer.GetNodeId(); IPEndPoint containerManagerConnectAddress = NetUtils.CreateSocketAddrForHost(node .GetHost(), node.GetPort()); YarnRPC rpc = GetYarnRPC(); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(containerId .GetApplicationAttemptId().ToString()); string user = rmContext.GetRMApps()[containerId.GetApplicationAttemptId().GetApplicationId ()].GetUser(); Token token = rmContext.GetNMTokenSecretManager().CreateNMToken(containerId.GetApplicationAttemptId (), node, user); currentUser.AddToken(ConverterUtils.ConvertFromYarn(token, containerManagerConnectAddress )); return(NMProxy.CreateNMProxy <ContainerManagementProtocol>(conf, currentUser, rpc, containerManagerConnectAddress)); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual RegisterNodeManagerResponse RegisterNodeManager(RegisterNodeManagerRequest request) { NodeId nodeId = request.GetNodeId(); string host = nodeId.GetHost(); int cmPort = nodeId.GetPort(); int httpPort = request.GetHttpPort(); Resource capability = request.GetResource(); string nodeManagerVersion = request.GetNMVersion(); RegisterNodeManagerResponse response = recordFactory.NewRecordInstance <RegisterNodeManagerResponse >(); if (!minimumNodeManagerVersion.Equals("NONE")) { if (minimumNodeManagerVersion.Equals("EqualToRM")) { minimumNodeManagerVersion = YarnVersionInfo.GetVersion(); } if ((nodeManagerVersion == null) || (VersionUtil.CompareVersions(nodeManagerVersion , minimumNodeManagerVersion)) < 0) { string message = "Disallowed NodeManager Version " + nodeManagerVersion + ", is less than the minimum version " + minimumNodeManagerVersion + " sending SHUTDOWN signal to " + "NodeManager."; Log.Info(message); response.SetDiagnosticsMessage(message); response.SetNodeAction(NodeAction.Shutdown); return(response); } } // Check if this node is a 'valid' node if (!this.nodesListManager.IsValidNode(host)) { string message = "Disallowed NodeManager from " + host + ", Sending SHUTDOWN signal to the NodeManager."; Log.Info(message); response.SetDiagnosticsMessage(message); response.SetNodeAction(NodeAction.Shutdown); return(response); } // Check if this node has minimum allocations if (capability.GetMemory() < minAllocMb || capability.GetVirtualCores() < minAllocVcores) { string message = "NodeManager from " + host + " doesn't satisfy minimum allocations, Sending SHUTDOWN" + " signal to the NodeManager."; Log.Info(message); response.SetDiagnosticsMessage(message); response.SetNodeAction(NodeAction.Shutdown); return(response); } response.SetContainerTokenMasterKey(containerTokenSecretManager.GetCurrentKey()); response.SetNMTokenMasterKey(nmTokenSecretManager.GetCurrentKey()); RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort, Resolve (host), capability, nodeManagerVersion); RMNode oldNode = this.rmContext.GetRMNodes().PutIfAbsent(nodeId, rmNode); if (oldNode == null) { this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeStartedEvent(nodeId , request.GetNMContainerStatuses(), request.GetRunningApplications())); } else { Log.Info("Reconnect from the node at: " + host); this.nmLivelinessMonitor.Unregister(nodeId); // Reset heartbeat ID since node just restarted. oldNode.ResetLastNodeHeartBeatResponse(); this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeReconnectEvent( nodeId, rmNode, request.GetRunningApplications(), request.GetNMContainerStatuses ())); } // On every node manager register we will be clearing NMToken keys if // present for any running application. this.nmTokenSecretManager.RemoveNodeKey(nodeId); this.nmLivelinessMonitor.Register(nodeId); // Handle received container status, this should be processed after new // RMNode inserted if (!rmContext.IsWorkPreservingRecoveryEnabled()) { if (!request.GetNMContainerStatuses().IsEmpty()) { Log.Info("received container statuses on node manager register :" + request.GetNMContainerStatuses ()); foreach (NMContainerStatus status in request.GetNMContainerStatuses()) { HandleNMContainerStatus(status, nodeId); } } } string message_1 = "NodeManager from node " + host + "(cmPort: " + cmPort + " httpPort: " + httpPort + ") " + "registered with capability: " + capability + ", assigned nodeId " + nodeId; Log.Info(message_1); response.SetNodeAction(NodeAction.Normal); response.SetRMIdentifier(ResourceManager.GetClusterTimeStamp()); response.SetRMVersion(YarnVersionInfo.GetVersion()); return(response); }