/// <exception cref="System.IO.IOException"/>
protected internal virtual void CheckRemoveLabelsFromNode(IDictionary <NodeId, ICollection
<string> > removeLabelsFromNode)
{
// check all labels being added existed
ICollection <string> knownLabels = labelCollections.Keys;
foreach (KeyValuePair <NodeId, ICollection <string> > entry in removeLabelsFromNode)
{
NodeId nodeId = entry.Key;
ICollection <string> labels = entry.Value;
if (!knownLabels.ContainsAll(labels))
{
string msg = "Not all labels being removed contained by known " + "label collections, please check"
+ ", removed labels=[" + StringUtils.Join(labels, ",") + "]";
Log.Error(msg);
throw new IOException(msg);
}
ICollection <string> originalLabels = null;
bool nodeExisted = false;
if (WildcardPort != nodeId.GetPort())
{
CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId);
if (nm != null)
{
originalLabels = nm.labels;
nodeExisted = true;
}
}
else
{
CommonNodeLabelsManager.Host host = nodeCollections[nodeId.GetHost()];
if (null != host)
{
originalLabels = host.labels;
nodeExisted = true;
}
}
if (!nodeExisted)
{
string msg = "Try to remove labels from NM=" + nodeId + ", but the NM doesn't existed";
Log.Error(msg);
throw new IOException(msg);
}
// the labels will never be null
if (labels.IsEmpty())
{
continue;
}
// originalLabels may be null,
// because when a Node is created, Node.labels can be null.
if (originalLabels == null || !originalLabels.ContainsAll(labels))
{
string msg = "Try to remove labels = [" + StringUtils.Join(labels, ",") + "], but not all labels contained by NM="
+ nodeId;
Log.Error(msg);
throw new IOException(msg);
}
}
}
public static Token NewContainerToken(NodeId nodeId, byte[] password, ContainerTokenIdentifier
tokenIdentifier)
{
// RPC layer client expects ip:port as service for tokens
IPEndPoint addr = NetUtils.CreateSocketAddrForHost(nodeId.GetHost(), nodeId.GetPort
());
// NOTE: use SecurityUtil.setTokenService if this becomes a "real" token
Token containerToken = NewToken <Token>(tokenIdentifier.GetBytes(), ContainerTokenIdentifier
.Kind.ToString(), password, SecurityUtil.BuildTokenService(addr).ToString());
return(containerToken);
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual void CreateNodeIfNonExisted(NodeId nodeId)
{
CommonNodeLabelsManager.Host host = nodeCollections[nodeId.GetHost()];
if (null == host)
{
throw new IOException("Should create host before creating node.");
}
CommonNodeLabelsManager.Node nm = host.nms[nodeId];
if (null == nm)
{
host.nms[nodeId] = new CommonNodeLabelsManager.Node(nodeId);
}
}
public static Org.Apache.Hadoop.Yarn.Api.Records.Token NewInstance(byte[] password
, NMTokenIdentifier identifier)
{
NodeId nodeId = identifier.GetNodeId();
// RPC layer client expects ip:port as service for tokens
IPEndPoint addr = NetUtils.CreateSocketAddrForHost(nodeId.GetHost(), nodeId.GetPort
());
Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = Org.Apache.Hadoop.Yarn.Api.Records.Token
.NewInstance(identifier.GetBytes(), NMTokenIdentifier.Kind.ToString(), password,
SecurityUtil.BuildTokenService(addr).ToString());
return(nmToken);
}
protected internal virtual ContainerManagementProtocol GetContainerManagementProtocolProxy
(YarnRPC rpc, Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken, NodeId nodeId, string
user)
{
ContainerManagementProtocol proxy;
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);
IPEndPoint addr = NetUtils.CreateSocketAddr(nodeId.GetHost(), nodeId.GetPort());
if (nmToken != null)
{
ugi.AddToken(ConverterUtils.ConvertFromYarn(nmToken, addr));
}
proxy = NMProxy.CreateNMProxy <ContainerManagementProtocol>(conf, ugi, rpc, addr);
return(proxy);
}
/*
* Following methods are used for setting if a node is up and running, and it
* will update running nodes resource
*/
public virtual void ActivateNode(NodeId nodeId, Resource resource)
{
try
{
writeLock.Lock();
// save if we have a node before
IDictionary <string, CommonNodeLabelsManager.Host> before = CloneNodeMap(ImmutableSet
.Of(nodeId));
CreateHostIfNonExisted(nodeId.GetHost());
try
{
CreateNodeIfNonExisted(nodeId);
}
catch (IOException)
{
Log.Error("This shouldn't happen, cannot get host in nodeCollection" + " associated to the node being activated"
);
return;
}
CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId);
nm.resource = resource;
nm.running = true;
// Add node in labelsCollection
ICollection <string> labelsForNode = GetLabelsByNode(nodeId);
if (labelsForNode != null)
{
foreach (string label in labelsForNode)
{
NodeLabel labelInfo = labelCollections[label];
if (labelInfo != null)
{
labelInfo.AddNodeId(nodeId);
}
}
}
// get the node after edition
IDictionary <string, CommonNodeLabelsManager.Host> after = CloneNodeMap(ImmutableSet
.Of(nodeId));
UpdateResourceMappings(before, after);
}
finally
{
writeLock.Unlock();
}
}
protected internal virtual ICollection <string> GetLabelsByNode(NodeId nodeId, IDictionary
<string, CommonNodeLabelsManager.Host> map)
{
CommonNodeLabelsManager.Host host = map[nodeId.GetHost()];
if (null == host)
{
return(EmptyStringSet);
}
CommonNodeLabelsManager.Node nm = host.nms[nodeId];
if (null != nm && null != nm.labels)
{
return(nm.labels);
}
else
{
return(host.labels);
}
}
protected internal virtual CommonNodeLabelsManager.Node GetNMInNodeSet(NodeId nodeId
, IDictionary <string, CommonNodeLabelsManager.Host> map, bool checkRunning)
{
CommonNodeLabelsManager.Host host = map[nodeId.GetHost()];
if (null == host)
{
return(null);
}
CommonNodeLabelsManager.Node nm = host.nms[nodeId];
if (null == nm)
{
return(null);
}
if (checkRunning)
{
return(nm.running ? nm : null);
}
return(nm);
}
public static FiCaSchedulerNode GetMockNode(string host, string rack, int port, int
capability)
{
NodeId nodeId = Org.Mockito.Mockito.Mock <NodeId>();
Org.Mockito.Mockito.When(nodeId.GetHost()).ThenReturn(host);
Org.Mockito.Mockito.When(nodeId.GetPort()).ThenReturn(port);
RMNode rmNode = Org.Mockito.Mockito.Mock <RMNode>();
Org.Mockito.Mockito.When(rmNode.GetNodeID()).ThenReturn(nodeId);
Org.Mockito.Mockito.When(rmNode.GetTotalCapability()).ThenReturn(Resources.CreateResource
(capability, 1));
Org.Mockito.Mockito.When(rmNode.GetNodeAddress()).ThenReturn(host + ":" + port);
Org.Mockito.Mockito.When(rmNode.GetHostName()).ThenReturn(host);
Org.Mockito.Mockito.When(rmNode.GetRackName()).ThenReturn(rack);
FiCaSchedulerNode node = Org.Mockito.Mockito.Spy(new FiCaSchedulerNode(rmNode, false
));
Log.Info("node = " + host + " avail=" + node.GetAvailableResource());
return(node);
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual void CheckAddLabelsToNode(IDictionary <NodeId, ICollection
<string> > addedLabelsToNode)
{
if (null == addedLabelsToNode || addedLabelsToNode.IsEmpty())
{
return;
}
// check all labels being added existed
ICollection <string> knownLabels = labelCollections.Keys;
foreach (KeyValuePair <NodeId, ICollection <string> > entry in addedLabelsToNode)
{
NodeId nodeId = entry.Key;
ICollection <string> labels = entry.Value;
if (!knownLabels.ContainsAll(labels))
{
string msg = "Not all labels being added contained by known " + "label collections, please check"
+ ", added labels=[" + StringUtils.Join(labels, ",") + "]";
Log.Error(msg);
throw new IOException(msg);
}
// In YARN-2694, we temporarily disable user add more than 1 labels on a
// same host
if (!labels.IsEmpty())
{
ICollection <string> newLabels = new HashSet <string>(GetLabelsByNode(nodeId));
Sharpen.Collections.AddAll(newLabels, labels);
// we don't allow number of labels on a node > 1 after added labels
if (newLabels.Count > 1)
{
string msg = string.Format("%d labels specified on host=%s after add labels to node"
+ ", please note that we do not support specifying multiple" + " labels on a single host for now."
, newLabels.Count, nodeId.GetHost());
Log.Error(msg);
throw new IOException(msg);
}
}
}
}
/*
* Following methods are used for setting if a node unregistered to RM
*/
public virtual void DeactivateNode(NodeId nodeId)
{
try
{
writeLock.Lock();
// save if we have a node before
IDictionary <string, CommonNodeLabelsManager.Host> before = CloneNodeMap(ImmutableSet
.Of(nodeId));
CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId);
if (null != nm)
{
if (null == nm.labels)
{
// When node deactivated, remove the nm from node collection if no
// labels explicitly set for this particular nm
// Save labels first, we need to remove label->nodes relation later
ICollection <string> savedNodeLabels = GetLabelsOnNode(nodeId);
// Remove this node in nodes collection
Sharpen.Collections.Remove(nodeCollections[nodeId.GetHost()].nms, nodeId);
// Remove this node in labels->node
RemoveNodeFromLabels(nodeId, savedNodeLabels);
}
else
{
// set nm is not running, and its resource = 0
nm.running = false;
nm.resource = Resource.NewInstance(0, 0);
}
}
// get the node after edition
IDictionary <string, CommonNodeLabelsManager.Host> after = CloneNodeMap(ImmutableSet
.Of(nodeId));
UpdateResourceMappings(before, after);
}
finally
{
writeLock.Unlock();
}
}
public static ImmutableMap <string, ICollection <NodeId> > TransposeNodeToLabels(IDictionary
<NodeId, ICollection <string> > mapNodeToLabels)
{
IDictionary <string, ICollection <NodeId> > mapLabelsToNodes = new Dictionary <string
, ICollection <NodeId> >();
foreach (KeyValuePair <NodeId, ICollection <string> > entry in mapNodeToLabels)
{
NodeId node = entry.Key;
ICollection <string> setLabels = entry.Value;
foreach (string label in setLabels)
{
ICollection <NodeId> setNode = mapLabelsToNodes[label];
if (setNode == null)
{
setNode = new HashSet <NodeId>();
}
setNode.AddItem(NodeId.NewInstance(node.GetHost(), node.GetPort()));
mapLabelsToNodes[label] = setNode;
}
}
return(ImmutableMap.CopyOf(mapLabelsToNodes));
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual void InternalUpdateLabelsOnNodes(IDictionary <NodeId, ICollection
<string> > nodeToLabels, CommonNodeLabelsManager.NodeLabelUpdateOperation op)
{
// do update labels from nodes
IDictionary <NodeId, ICollection <string> > newNMToLabels = new Dictionary <NodeId, ICollection
<string> >();
ICollection <string> oldLabels;
foreach (KeyValuePair <NodeId, ICollection <string> > entry in nodeToLabels)
{
NodeId nodeId = entry.Key;
ICollection <string> labels = entry.Value;
CreateHostIfNonExisted(nodeId.GetHost());
if (nodeId.GetPort() == WildcardPort)
{
CommonNodeLabelsManager.Host host = nodeCollections[nodeId.GetHost()];
switch (op)
{
case CommonNodeLabelsManager.NodeLabelUpdateOperation.Remove:
{
RemoveNodeFromLabels(nodeId, labels);
host.labels.RemoveAll(labels);
foreach (CommonNodeLabelsManager.Node node in host.nms.Values)
{
if (node.labels != null)
{
node.labels.RemoveAll(labels);
}
RemoveNodeFromLabels(node.nodeId, labels);
}
break;
}
case CommonNodeLabelsManager.NodeLabelUpdateOperation.Add:
{
AddNodeToLabels(nodeId, labels);
Sharpen.Collections.AddAll(host.labels, labels);
foreach (CommonNodeLabelsManager.Node node_1 in host.nms.Values)
{
if (node_1.labels != null)
{
Sharpen.Collections.AddAll(node_1.labels, labels);
}
AddNodeToLabels(node_1.nodeId, labels);
}
break;
}
case CommonNodeLabelsManager.NodeLabelUpdateOperation.Replace:
{
ReplaceNodeForLabels(nodeId, host.labels, labels);
host.labels.Clear();
Sharpen.Collections.AddAll(host.labels, labels);
foreach (CommonNodeLabelsManager.Node node_2 in host.nms.Values)
{
ReplaceNodeForLabels(node_2.nodeId, node_2.labels, labels);
node_2.labels = null;
}
break;
}
default:
{
break;
}
}
newNMToLabels[nodeId] = host.labels;
}
else
{
if (EnumSet.Of(CommonNodeLabelsManager.NodeLabelUpdateOperation.Add, CommonNodeLabelsManager.NodeLabelUpdateOperation
.Replace).Contains(op))
{
// Add and replace
CreateNodeIfNonExisted(nodeId);
CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId);
switch (op)
{
case CommonNodeLabelsManager.NodeLabelUpdateOperation.Add:
{
AddNodeToLabels(nodeId, labels);
if (nm.labels == null)
{
nm.labels = new HashSet <string>();
}
Sharpen.Collections.AddAll(nm.labels, labels);
break;
}
case CommonNodeLabelsManager.NodeLabelUpdateOperation.Replace:
{
oldLabels = GetLabelsByNode(nodeId);
ReplaceNodeForLabels(nodeId, oldLabels, labels);
if (nm.labels == null)
{
nm.labels = new HashSet <string>();
}
nm.labels.Clear();
Sharpen.Collections.AddAll(nm.labels, labels);
break;
}
default:
{
break;
}
}
newNMToLabels[nodeId] = nm.labels;
}
else
{
// remove
RemoveNodeFromLabels(nodeId, labels);
CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId);
if (nm.labels != null)
{
nm.labels.RemoveAll(labels);
newNMToLabels[nodeId] = nm.labels;
}
}
}
}
if (null != dispatcher)
{
dispatcher.GetEventHandler().Handle(new UpdateNodeToLabelsMappingsEvent(newNMToLabels
));
}
// shows node->labels we added
Log.Info(op.ToString() + " labels on nodes:");
foreach (KeyValuePair <NodeId, ICollection <string> > entry_1 in newNMToLabels)
{
Log.Info(" NM=" + entry_1.Key + ", labels=[" + StringUtils.Join(entry_1.Value.GetEnumerator
(), ",") + "]");
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual RegisterNodeManagerResponse RegisterNodeManager(RegisterNodeManagerRequest
request)
{
NodeId nodeId = request.GetNodeId();
string host = nodeId.GetHost();
int cmPort = nodeId.GetPort();
int httpPort = request.GetHttpPort();
Resource capability = request.GetResource();
string nodeManagerVersion = request.GetNMVersion();
RegisterNodeManagerResponse response = recordFactory.NewRecordInstance <RegisterNodeManagerResponse
>();
if (!minimumNodeManagerVersion.Equals("NONE"))
{
if (minimumNodeManagerVersion.Equals("EqualToRM"))
{
minimumNodeManagerVersion = YarnVersionInfo.GetVersion();
}
if ((nodeManagerVersion == null) || (VersionUtil.CompareVersions(nodeManagerVersion
, minimumNodeManagerVersion)) < 0)
{
string message = "Disallowed NodeManager Version " + nodeManagerVersion + ", is less than the minimum version "
+ minimumNodeManagerVersion + " sending SHUTDOWN signal to " + "NodeManager.";
Log.Info(message);
response.SetDiagnosticsMessage(message);
response.SetNodeAction(NodeAction.Shutdown);
return(response);
}
}
// Check if this node is a 'valid' node
if (!this.nodesListManager.IsValidNode(host))
{
string message = "Disallowed NodeManager from " + host + ", Sending SHUTDOWN signal to the NodeManager.";
Log.Info(message);
response.SetDiagnosticsMessage(message);
response.SetNodeAction(NodeAction.Shutdown);
return(response);
}
// Check if this node has minimum allocations
if (capability.GetMemory() < minAllocMb || capability.GetVirtualCores() < minAllocVcores)
{
string message = "NodeManager from " + host + " doesn't satisfy minimum allocations, Sending SHUTDOWN"
+ " signal to the NodeManager.";
Log.Info(message);
response.SetDiagnosticsMessage(message);
response.SetNodeAction(NodeAction.Shutdown);
return(response);
}
response.SetContainerTokenMasterKey(containerTokenSecretManager.GetCurrentKey());
response.SetNMTokenMasterKey(nmTokenSecretManager.GetCurrentKey());
RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort, Resolve
(host), capability, nodeManagerVersion);
RMNode oldNode = this.rmContext.GetRMNodes().PutIfAbsent(nodeId, rmNode);
if (oldNode == null)
{
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeStartedEvent(nodeId
, request.GetNMContainerStatuses(), request.GetRunningApplications()));
}
else
{
Log.Info("Reconnect from the node at: " + host);
this.nmLivelinessMonitor.Unregister(nodeId);
// Reset heartbeat ID since node just restarted.
oldNode.ResetLastNodeHeartBeatResponse();
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeReconnectEvent(
nodeId, rmNode, request.GetRunningApplications(), request.GetNMContainerStatuses
()));
}
// On every node manager register we will be clearing NMToken keys if
// present for any running application.
this.nmTokenSecretManager.RemoveNodeKey(nodeId);
this.nmLivelinessMonitor.Register(nodeId);
// Handle received container status, this should be processed after new
// RMNode inserted
if (!rmContext.IsWorkPreservingRecoveryEnabled())
{
if (!request.GetNMContainerStatuses().IsEmpty())
{
Log.Info("received container statuses on node manager register :" + request.GetNMContainerStatuses
());
foreach (NMContainerStatus status in request.GetNMContainerStatuses())
{
HandleNMContainerStatus(status, nodeId);
}
}
}
string message_1 = "NodeManager from node " + host + "(cmPort: " + cmPort + " httpPort: "
+ httpPort + ") " + "registered with capability: " + capability + ", assigned nodeId "
+ nodeId;
Log.Info(message_1);
response.SetNodeAction(NodeAction.Normal);
response.SetRMIdentifier(ResourceManager.GetClusterTimeStamp());
response.SetRMVersion(YarnVersionInfo.GetVersion());
return(response);
}
// Protected. For tests.
protected internal virtual ContainerManagementProtocol GetContainerMgrProxy(ContainerId
containerId)
{
NodeId node = masterContainer.GetNodeId();
IPEndPoint containerManagerConnectAddress = NetUtils.CreateSocketAddrForHost(node
.GetHost(), node.GetPort());
YarnRPC rpc = GetYarnRPC();
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(containerId
.GetApplicationAttemptId().ToString());
string user = rmContext.GetRMApps()[containerId.GetApplicationAttemptId().GetApplicationId
()].GetUser();
Token token = rmContext.GetNMTokenSecretManager().CreateNMToken(containerId.GetApplicationAttemptId
(), node, user);
currentUser.AddToken(ConverterUtils.ConvertFromYarn(token, containerManagerConnectAddress
));
return(NMProxy.CreateNMProxy <ContainerManagementProtocol>(conf, currentUser, rpc,
containerManagerConnectAddress));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual NodeHeartbeatResponse NodeHeartbeat(NodeHeartbeatRequest request)
{
NodeStatus remoteNodeStatus = request.GetNodeStatus();
NodeId nodeId = remoteNodeStatus.GetNodeId();
// 1. Check if it's a valid (i.e. not excluded) node
if (!this.nodesListManager.IsValidNode(nodeId.GetHost()))
{
string message = "Disallowed NodeManager nodeId: " + nodeId + " hostname: " + nodeId
.GetHost();
Log.Info(message);
shutDown.SetDiagnosticsMessage(message);
return(shutDown);
}
// 2. Check if it's a registered node
RMNode rmNode = this.rmContext.GetRMNodes()[nodeId];
if (rmNode == null)
{
/* node does not exist */
string message = "Node not found resyncing " + remoteNodeStatus.GetNodeId();
Log.Info(message);
resync.SetDiagnosticsMessage(message);
return(resync);
}
// Send ping
this.nmLivelinessMonitor.ReceivedPing(nodeId);
// 3. Check if it's a 'fresh' heartbeat i.e. not duplicate heartbeat
NodeHeartbeatResponse lastNodeHeartbeatResponse = rmNode.GetLastNodeHeartBeatResponse
();
if (remoteNodeStatus.GetResponseId() + 1 == lastNodeHeartbeatResponse.GetResponseId
())
{
Log.Info("Received duplicate heartbeat from node " + rmNode.GetNodeAddress() + " responseId="
+ remoteNodeStatus.GetResponseId());
return(lastNodeHeartbeatResponse);
}
else
{
if (remoteNodeStatus.GetResponseId() + 1 < lastNodeHeartbeatResponse.GetResponseId
())
{
string message = "Too far behind rm response id:" + lastNodeHeartbeatResponse.GetResponseId
() + " nm response id:" + remoteNodeStatus.GetResponseId();
Log.Info(message);
resync.SetDiagnosticsMessage(message);
// TODO: Just sending reboot is not enough. Think more.
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeEvent(nodeId, RMNodeEventType
.Rebooting));
return(resync);
}
}
// Heartbeat response
NodeHeartbeatResponse nodeHeartBeatResponse = YarnServerBuilderUtils.NewNodeHeartbeatResponse
(lastNodeHeartbeatResponse.GetResponseId() + 1, NodeAction.Normal, null, null, null
, null, nextHeartBeatInterval);
rmNode.UpdateNodeHeartbeatResponseForCleanup(nodeHeartBeatResponse);
PopulateKeys(request, nodeHeartBeatResponse);
ConcurrentMap <ApplicationId, ByteBuffer> systemCredentials = rmContext.GetSystemCredentialsForApps
();
if (!systemCredentials.IsEmpty())
{
nodeHeartBeatResponse.SetSystemCredentialsForApps(systemCredentials);
}
// 4. Send status to RMNode, saving the latest response.
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeStatusEvent(nodeId
, remoteNodeStatus.GetNodeHealthStatus(), remoteNodeStatus.GetContainersStatuses
(), remoteNodeStatus.GetKeepAliveApplications(), nodeHeartBeatResponse));
return(nodeHeartBeatResponse);
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual void CheckReplaceLabelsOnNode(IDictionary <NodeId, ICollection
<string> > replaceLabelsToNode)
{
if (null == replaceLabelsToNode || replaceLabelsToNode.IsEmpty())
{
return;
}
// check all labels being added existed
ICollection <string> knownLabels = labelCollections.Keys;
foreach (KeyValuePair <NodeId, ICollection <string> > entry in replaceLabelsToNode)
{
NodeId nodeId = entry.Key;
ICollection <string> labels = entry.Value;
// As in YARN-2694, we disable user add more than 1 labels on a same host
if (labels.Count > 1)
{
string msg = string.Format("%d labels specified on host=%s" + ", please note that we do not support specifying multiple"
+ " labels on a single host for now.", labels.Count, nodeId.GetHost());
Log.Error(msg);
throw new IOException(msg);
}
if (!knownLabels.ContainsAll(labels))
{
string msg = "Not all labels being replaced contained by known " + "label collections, please check"
+ ", new labels=[" + StringUtils.Join(labels, ",") + "]";
Log.Error(msg);
throw new IOException(msg);
}
}
}
