/// <exception cref="System.Exception"/>
public virtual void TestRemoveLabelsFromNode()
{
mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3"));
mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n2"
), ToSet("p2"), ToNodeId("n3"), ToSet("p3")));
// active one NM to n1:1
mgr.ActivateNode(NodeId.NewInstance("n1", 1), SmallResource);
try
{
mgr.RemoveLabelsFromNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p1")));
NUnit.Framework.Assert.Fail("removeLabelsFromNode should trigger IOException");
}
catch (IOException)
{
}
mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p1")));
try
{
mgr.RemoveLabelsFromNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p1")));
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("IOException from removeLabelsFromNode " + e);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestLabelsToNodesOnNodeActiveDeactive()
{
// Activate a node without assigning any labels
mgr.ActivateNode(NodeId.NewInstance("n1", 1), Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(10, 0));
NUnit.Framework.Assert.IsTrue(mgr.GetLabelsToNodes().IsEmpty());
AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), TransposeNodeToLabels(mgr.GetNodeLabels
()));
// Add labels and replace labels on node
mgr.AddToCluserNodeLabels(ToSet("p1"));
mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1")));
// p1 -> n1, n1:1
NUnit.Framework.Assert.AreEqual(2, mgr.GetLabelsToNodes()["p1"].Count);
AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), TransposeNodeToLabels(mgr.GetNodeLabels
()));
// Activate a node for which host to label mapping exists
mgr.ActivateNode(NodeId.NewInstance("n1", 2), Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(10, 0));
// p1 -> n1, n1:1, n1:2
NUnit.Framework.Assert.AreEqual(3, mgr.GetLabelsToNodes()["p1"].Count);
AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), TransposeNodeToLabels(mgr.GetNodeLabels
()));
// Deactivate a node. n1:1 will be removed from the map
mgr.DeactivateNode(NodeId.NewInstance("n1", 1));
// p1 -> n1, n1:2
NUnit.Framework.Assert.AreEqual(2, mgr.GetLabelsToNodes()["p1"].Count);
AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), TransposeNodeToLabels(mgr.GetNodeLabels
()));
}
public AMAttemptInfo(AMInfo amInfo, string jobId, string user)
{
this.nodeHttpAddress = string.Empty;
this.nodeId = string.Empty;
string nmHost = amInfo.GetNodeManagerHost();
int nmHttpPort = amInfo.GetNodeManagerHttpPort();
int nmPort = amInfo.GetNodeManagerPort();
if (nmHost != null)
{
this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
NodeId nodeId = NodeId.NewInstance(nmHost, nmPort);
this.nodeId = nodeId.ToString();
}
this.id = amInfo.GetAppAttemptId().GetAttemptId();
this.startTime = amInfo.GetStartTime();
this.containerId = string.Empty;
this.logsLink = string.Empty;
ContainerId containerId = amInfo.GetContainerId();
if (containerId != null)
{
this.containerId = containerId.ToString();
this.logsLink = StringHelper.Join(MRWebAppUtil.GetYARNWebappScheme() + nodeHttpAddress
, StringHelper.Ujoin("node", "containerlogs", this.containerId, user));
}
}
/// <summary>Get mapping of nodes to labels</summary>
/// <returns>nodes to labels map</returns>
public virtual IDictionary <NodeId, ICollection <string> > GetNodeLabels()
{
try
{
readLock.Lock();
IDictionary <NodeId, ICollection <string> > nodeToLabels = new Dictionary <NodeId, ICollection
<string> >();
foreach (KeyValuePair <string, CommonNodeLabelsManager.Host> entry in nodeCollections)
{
string hostName = entry.Key;
CommonNodeLabelsManager.Host host = entry.Value;
foreach (NodeId nodeId in host.nms.Keys)
{
ICollection <string> nodeLabels = GetLabelsByNode(nodeId);
if (nodeLabels == null || nodeLabels.IsEmpty())
{
continue;
}
nodeToLabels[nodeId] = nodeLabels;
}
if (!host.labels.IsEmpty())
{
nodeToLabels[NodeId.NewInstance(hostName, WildcardPort)] = host.labels;
}
}
return(Sharpen.Collections.UnmodifiableMap(nodeToLabels));
}
finally
{
readLock.Unlock();
}
}
public virtual void Handle(ContainerAllocatorEvent @event)
{
if (@event.GetType() == ContainerAllocator.EventType.ContainerReq)
{
Log.Info("Processing the event " + @event.ToString());
// Assign the same container ID as the AM
ContainerId cID = ContainerId.NewContainerId(GetContext().GetApplicationAttemptId
(), this.containerId.GetContainerId());
Container container = recordFactory.NewRecordInstance <Container>();
container.SetId(cID);
NodeId nodeId = NodeId.NewInstance(this.nmHost, this.nmPort);
container.SetNodeId(nodeId);
container.SetContainerToken(null);
container.SetNodeHttpAddress(this.nmHost + ":" + this.nmHttpPort);
// send the container-assigned event to task attempt
if (@event.GetAttemptID().GetTaskId().GetTaskType() == TaskType.Map)
{
JobCounterUpdateEvent jce = new JobCounterUpdateEvent(@event.GetAttemptID().GetTaskId
().GetJobId());
// TODO Setting OTHER_LOCAL_MAP for now.
jce.AddCounterUpdate(JobCounter.OtherLocalMaps, 1);
eventHandler.Handle(jce);
}
eventHandler.Handle(new TaskAttemptContainerAssignedEvent(@event.GetAttemptID(),
container, applicationACLs));
}
}
/// <exception cref="System.Exception"/>
public virtual void TestActivateNodeManagerWithZeroPort()
{
// active two NM, one is zero port , another is non-zero port. no exception
// should be raised
mgr.ActivateNode(NodeId.NewInstance("n1", 0), SmallResource);
mgr.ActivateNode(NodeId.NewInstance("n1", 2), LargeNode);
}
public virtual void TestLogsViewSingle()
{
Log.Info("HsLogsPage with params for single log and data limits");
MockAppContext ctx = new MockAppContext(0, 1, 1, 1);
IDictionary <string, string> @params = new Dictionary <string, string>();
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, true);
@params["start"] = "-2048";
@params["end"] = "-1024";
@params[YarnWebParams.ContainerLogType] = "syslog";
@params[YarnWebParams.ContainerId] = MRApp.NewContainerId(1, 1, 333, 1).ToString(
);
@params[YarnWebParams.NmNodename] = NodeId.NewInstance(MockJobs.NmHost, MockJobs.
NmPort).ToString();
@params[YarnWebParams.EntityString] = "container_10_0001_01_000001";
@params[YarnWebParams.AppOwner] = "owner";
Injector injector = WebAppTests.TestPage <AppContext>(typeof(AggregatedLogsPage),
ctx, @params, new _AbstractModule_201(conf));
PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);
Org.Mockito.Mockito.Verify(spyPw).Write("Logs not available for container_10_0001_01_000001."
+ " Aggregation may not be complete, " + "Check back later or try the nodemanager at "
+ MockJobs.NmHost + ":" + MockJobs.NmPort);
}
public virtual void TestNMTokenIdentifier()
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(ApplicationId
.NewInstance(1, 1), 1);
NodeId nodeId = NodeId.NewInstance("host0", 0);
string applicationSubmitter = "usr0";
int masterKeyId = 1;
NMTokenIdentifier token = new NMTokenIdentifier(appAttemptId, nodeId, applicationSubmitter
, masterKeyId);
NMTokenIdentifier anotherToken = new NMTokenIdentifier();
byte[] tokenContent = token.GetBytes();
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenContent, tokenContent.Length);
anotherToken.ReadFields(dib);
// verify the whole record equals with original record
NUnit.Framework.Assert.AreEqual("Token is not the same after serialization " + "and deserialization."
, token, anotherToken);
// verify all properties are the same as original
NUnit.Framework.Assert.AreEqual("appAttemptId from proto is not the same with original token"
, anotherToken.GetApplicationAttemptId(), appAttemptId);
NUnit.Framework.Assert.AreEqual("NodeId from proto is not the same with original token"
, anotherToken.GetNodeId(), nodeId);
NUnit.Framework.Assert.AreEqual("applicationSubmitter from proto is not the same with original token"
, anotherToken.GetApplicationSubmitter(), applicationSubmitter);
NUnit.Framework.Assert.AreEqual("masterKeyId from proto is not the same with original token"
, anotherToken.GetKeyId(), masterKeyId);
}
private static RMContainer CreateRMContainer(ContainerId containerId)
{
RMContainer container = Org.Mockito.Mockito.Mock <RMContainer>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(containerId);
Org.Mockito.Mockito.When(container.GetAllocatedNode()).ThenReturn(NodeId.NewInstance
("test host", -100));
Org.Mockito.Mockito.When(container.GetAllocatedResource()).ThenReturn(Resource.NewInstance
(-1, -1));
Org.Mockito.Mockito.When(container.GetAllocatedPriority()).ThenReturn(Priority.Undefined
);
Org.Mockito.Mockito.When(container.GetCreationTime()).ThenReturn(int.MaxValue + 1L
);
Org.Mockito.Mockito.When(container.GetFinishTime()).ThenReturn(int.MaxValue + 2L);
Org.Mockito.Mockito.When(container.GetDiagnosticsInfo()).ThenReturn("test diagnostics info"
);
Org.Mockito.Mockito.When(container.GetContainerExitStatus()).ThenReturn(-1);
Org.Mockito.Mockito.When(container.GetContainerState()).ThenReturn(ContainerState
.Complete);
Container mockContainer = Org.Mockito.Mockito.Mock <Container>();
Org.Mockito.Mockito.When(container.GetContainer()).ThenReturn(mockContainer);
Org.Mockito.Mockito.When(mockContainer.GetNodeHttpAddress()).ThenReturn("http://localhost:1234"
);
return(container);
}
public virtual void TestReconnect()
{
string hostname1 = "localhost1";
Resource capability = BuilderUtils.NewResource(1024, 1);
RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
>();
NodeId nodeId1 = NodeId.NewInstance(hostname1, 0);
request1.SetNodeId(nodeId1);
request1.SetHttpPort(0);
request1.SetResource(capability);
resourceTrackerService.RegisterNodeManager(request1);
NUnit.Framework.Assert.AreEqual(RMNodeEventType.Started, rmNodeEvents[0].GetType(
));
rmNodeEvents.Clear();
resourceTrackerService.RegisterNodeManager(request1);
NUnit.Framework.Assert.AreEqual(RMNodeEventType.Reconnected, rmNodeEvents[0].GetType
());
rmNodeEvents.Clear();
resourceTrackerService.RegisterNodeManager(request1);
capability = BuilderUtils.NewResource(1024, 2);
request1.SetResource(capability);
NUnit.Framework.Assert.AreEqual(RMNodeEventType.Reconnected, rmNodeEvents[0].GetType
());
}
public AMAttemptInfo(AMInfo amInfo, string jobId, string user, string host, string
pathPrefix)
{
this.nodeHttpAddress = string.Empty;
this.nodeId = string.Empty;
string nmHost = amInfo.GetNodeManagerHost();
int nmHttpPort = amInfo.GetNodeManagerHttpPort();
int nmPort = amInfo.GetNodeManagerPort();
if (nmHost != null)
{
this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
NodeId nodeId = NodeId.NewInstance(nmHost, nmPort);
this.nodeId = nodeId.ToString();
}
this.id = amInfo.GetAppAttemptId().GetAttemptId();
this.startTime = amInfo.GetStartTime();
this.containerId = string.Empty;
this.logsLink = string.Empty;
this.shortLogsLink = string.Empty;
ContainerId containerId = amInfo.GetContainerId();
if (containerId != null)
{
this.containerId = containerId.ToString();
this.logsLink = StringHelper.Join(host, pathPrefix, StringHelper.Ujoin("logs", this
.nodeId, this.containerId, jobId, user));
this.shortLogsLink = StringHelper.Ujoin("logs", this.nodeId, this.containerId, jobId
, user);
}
}
public virtual void TestNodeRegistrationVersionLessThanRM()
{
WriteToHostsFile("host2");
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
conf.Set(YarnConfiguration.RmNodemanagerMinimumVersion, "EqualToRM");
rm = new MockRM(conf);
rm.Start();
string nmVersion = "1.9.9";
ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService();
RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
>();
NodeId nodeId = NodeId.NewInstance("host2", 1234);
Resource capability = BuilderUtils.NewResource(1024, 1);
req.SetResource(capability);
req.SetNodeId(nodeId);
req.SetHttpPort(1234);
req.SetNMVersion(nmVersion);
// trying to register a invalid node.
RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager
(req);
NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response.GetNodeAction());
NUnit.Framework.Assert.IsTrue("Diagnostic message did not contain: 'Disallowed NodeManager "
+ "Version " + nmVersion + ", is less than the minimum version'", response.GetDiagnosticsMessage
().Contains("Disallowed NodeManager Version " + nmVersion + ", is less than the minimum version "
));
}
public virtual void TestUpdateMaxAllocationUsesTotal()
{
int configuredMaxVCores = 20;
int configuredMaxMemory = 10 * 1024;
Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(configuredMaxMemory, configuredMaxVCores);
ConfigureScheduler();
YarnConfiguration conf = GetConf();
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
);
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
);
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
MockRM rm = new MockRM(conf);
try
{
rm.Start();
AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler(
);
Org.Apache.Hadoop.Yarn.Api.Records.Resource emptyResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(0, 0);
Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 5);
Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 10);
SchedulerNode mockNode1 = Org.Mockito.Mockito.Mock <SchedulerNode>();
Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("foo"
, 8080));
Org.Mockito.Mockito.When(mockNode1.GetAvailableResource()).ThenReturn(emptyResource
);
Org.Mockito.Mockito.When(mockNode1.GetTotalResource()).ThenReturn(fullResource1);
SchedulerNode mockNode2 = Org.Mockito.Mockito.Mock <SchedulerNode>();
Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("bar"
, 8081));
Org.Mockito.Mockito.When(mockNode2.GetAvailableResource()).ThenReturn(emptyResource
);
Org.Mockito.Mockito.When(mockNode2.GetTotalResource()).ThenReturn(fullResource2);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
scheduler.nodes = new Dictionary <NodeId, SchedulerNode>();
scheduler.nodes[mockNode1.GetNodeID()] = mockNode1;
scheduler.UpdateMaximumAllocation(mockNode1, true);
VerifyMaximumResourceCapability(fullResource1, scheduler);
scheduler.nodes[mockNode2.GetNodeID()] = mockNode2;
scheduler.UpdateMaximumAllocation(mockNode2, true);
VerifyMaximumResourceCapability(fullResource2, scheduler);
Sharpen.Collections.Remove(scheduler.nodes, mockNode2.GetNodeID());
scheduler.UpdateMaximumAllocation(mockNode2, false);
VerifyMaximumResourceCapability(fullResource1, scheduler);
Sharpen.Collections.Remove(scheduler.nodes, mockNode1.GetNodeID());
scheduler.UpdateMaximumAllocation(mockNode1, false);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
}
finally
{
rm.Stop();
}
}
public virtual void TestRegisterNodeManagerRequest()
{
ApplicationId appId = ApplicationId.NewInstance(123456789, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId containerId = ContainerId.NewContainerId(attemptId, 1);
NMContainerStatus containerReport = NMContainerStatus.NewInstance(containerId, ContainerState
.Running, Resource.NewInstance(1024, 1), "diagnostics", 0, Priority.NewInstance(
10), 1234);
IList <NMContainerStatus> reports = Arrays.AsList(containerReport);
RegisterNodeManagerRequest request = RegisterNodeManagerRequest.NewInstance(NodeId
.NewInstance("1.1.1.1", 1000), 8080, Resource.NewInstance(1024, 1), "NM-version-id"
, reports, Arrays.AsList(appId));
RegisterNodeManagerRequest requestProto = new RegisterNodeManagerRequestPBImpl(((
RegisterNodeManagerRequestPBImpl)request).GetProto());
NUnit.Framework.Assert.AreEqual(containerReport, requestProto.GetNMContainerStatuses
()[0]);
NUnit.Framework.Assert.AreEqual(8080, requestProto.GetHttpPort());
NUnit.Framework.Assert.AreEqual("NM-version-id", requestProto.GetNMVersion());
NUnit.Framework.Assert.AreEqual(NodeId.NewInstance("1.1.1.1", 1000), requestProto
.GetNodeId());
NUnit.Framework.Assert.AreEqual(Resource.NewInstance(1024, 1), requestProto.GetResource
());
NUnit.Framework.Assert.AreEqual(1, requestProto.GetRunningApplications().Count);
NUnit.Framework.Assert.AreEqual(appId, requestProto.GetRunningApplications()[0]);
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerAllocateWithLabels()
{
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
.NewInstance("h2", 0), ToSet("y")));
// inject node label manager
MockRM rm1 = new _MockRM_638(this, GetConfigurationWithQueueLabels(conf));
rm1.GetRMContext().SetNodeLabelManager(mgr);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("h1:1234", 8000);
// label = x
MockNM nm2 = rm1.RegisterNode("h2:1234", 8000);
// label = y
MockNM nm3 = rm1.RegisterNode("h3:1234", 8000);
// label = <empty>
ContainerId containerId;
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm3);
// request a container.
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x");
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am1.GetApplicationAttemptId(), containerId, rm1, "h1");
// launch an app to queue b1 (label = y), and check all container will
// be allocated in h2
RMApp app2 = rm1.SubmitApp(200, "app", "user", null, "b1");
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm3);
// request a container.
am2.Allocate("*", 1024, 1, new AList <ContainerId>(), "y");
containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am2.GetApplicationAttemptId(), containerId, rm1, "h2");
// launch an app to queue c1 (label = ""), and check all container will
// be allocated in h3
RMApp app3 = rm1.SubmitApp(200, "app", "user", null, "c1");
MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm3);
// request a container.
am3.Allocate("*", 1024, 1, new AList <ContainerId>());
containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h3");
rm1.Close();
}
private static Container CreateMockContainer()
{
Container container = Org.Mockito.Mockito.Mock <Container>();
NodeId nodeId = NodeId.NewInstance("foo.bar.org", 1234);
Org.Mockito.Mockito.When(container.GetNodeId()).ThenReturn(nodeId);
return(container);
}
private Token CreateNewContainerToken(ContainerId contId, string containerManagerAddr
)
{
long currentTime = Runtime.CurrentTimeMillis();
return(MRApp.NewContainerToken(NodeId.NewInstance("127.0.0.1", 1234), Sharpen.Runtime.GetBytesForString
("password"), new ContainerTokenIdentifier(contId, containerManagerAddr, "user",
Resource.NewInstance(1024, 1), currentTime + 10000L, 123, currentTime, Priority.
NewInstance(0), 0)));
}
public virtual IList <NodeReport> CreateFakeNodeReports()
{
NodeId nodeId = NodeId.NewInstance("localhost", 0);
NodeReport report = NodeReport.NewInstance(nodeId, NodeState.Running, "localhost"
, "rack1", null, null, 4, null, 1000l, null);
IList <NodeReport> reports = new AList <NodeReport>();
reports.AddItem(report);
return(reports);
}
private IDictionary <string, CommonNodeLabelsManager.Host> CloneNodeMap()
{
ICollection <NodeId> nodesToCopy = new HashSet <NodeId>();
foreach (string nodeName in nodeCollections.Keys)
{
nodesToCopy.AddItem(NodeId.NewInstance(nodeName, WildcardPort));
}
return(CloneNodeMap(nodesToCopy));
}
/// <exception cref="System.Exception"/>
private void TestRPCTimeout(string rpcClass)
{
Configuration conf = new Configuration();
// set timeout low for the test
conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
try
{
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), server.GetListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
try
{
proxy.StartContainers(allRequests);
}
catch (Exception e)
{
Log.Info(StringUtils.StringifyException(e));
NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException
).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName);
return;
}
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.Fail("timeout exception should have occurred!");
}
private Container MockContainer(int i)
{
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId containerId = ContainerId.NewContainerId(attemptId, i);
nodeId = NodeId.NewInstance("localhost", 0);
// Create an empty record
containerToken = recordFactory.NewRecordInstance <Token>();
return(Container.NewInstance(containerId, nodeId, null, null, null, containerToken
));
}
public virtual void TestCompareRMNodeAfterReconnect()
{
Configuration yarnConf = new YarnConfiguration();
CapacityScheduler scheduler = new CapacityScheduler();
scheduler.SetConf(yarnConf);
ConfigurationProvider configurationProvider = ConfigurationProviderFactory.GetConfigurationProvider
(yarnConf);
configurationProvider.Init(yarnConf);
context.SetConfigurationProvider(configurationProvider);
RMNodeLabelsManager nlm = new RMNodeLabelsManager();
nlm.Init(yarnConf);
nlm.Start();
context.SetNodeLabelManager(nlm);
scheduler.SetRMContext(context);
scheduler.Init(yarnConf);
scheduler.Start();
dispatcher.Register(typeof(SchedulerEventType), scheduler);
string hostname1 = "localhost1";
Resource capability = BuilderUtils.NewResource(4096, 4);
RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
>();
NodeId nodeId1 = NodeId.NewInstance(hostname1, 0);
request1.SetNodeId(nodeId1);
request1.SetHttpPort(0);
request1.SetResource(capability);
resourceTrackerService.RegisterNodeManager(request1);
NUnit.Framework.Assert.IsNotNull(context.GetRMNodes()[nodeId1]);
// verify Scheduler and RMContext use same RMNode reference.
NUnit.Framework.Assert.IsTrue(scheduler.GetSchedulerNode(nodeId1).GetRMNode() ==
context.GetRMNodes()[nodeId1]);
NUnit.Framework.Assert.AreEqual(context.GetRMNodes()[nodeId1].GetTotalCapability(
), capability);
Resource capability1 = BuilderUtils.NewResource(2048, 2);
request1.SetResource(capability1);
resourceTrackerService.RegisterNodeManager(request1);
NUnit.Framework.Assert.IsNotNull(context.GetRMNodes()[nodeId1]);
// verify Scheduler and RMContext use same RMNode reference
// after reconnect.
NUnit.Framework.Assert.IsTrue(scheduler.GetSchedulerNode(nodeId1).GetRMNode() ==
context.GetRMNodes()[nodeId1]);
// verify RMNode's capability is changed.
NUnit.Framework.Assert.AreEqual(context.GetRMNodes()[nodeId1].GetTotalCapability(
), capability1);
nlm.Stop();
scheduler.Stop();
}
GetCMProxy(string containerMgrBindAddr, ContainerId containerId)
{
IPEndPoint addr = NetUtils.GetConnectAddress(this._enclosing._enclosing.server);
string containerManagerBindAddr = addr.GetHostName() + ":" + addr.Port;
Token token = this._enclosing.tokenSecretManager.CreateNMToken(containerId.GetApplicationAttemptId
(), NodeId.NewInstance(addr.GetHostName(), addr.Port), "user");
ContainerManagementProtocolProxy cmProxy = new ContainerManagementProtocolProxy(this
._enclosing._enclosing.conf);
ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData proxy = new
ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData(this, YarnRPC
.Create(this._enclosing._enclosing.conf), containerManagerBindAddr, containerId,
token);
return proxy;
}
public virtual NodeId ToNodeId(string str)
{
if (str.Contains(":"))
{
int idx = str.IndexOf(':');
NodeId id = NodeId.NewInstance(Sharpen.Runtime.Substring(str, 0, idx), Sharpen.Extensions.ValueOf
(Sharpen.Runtime.Substring(str, idx + 1)));
return(id);
}
else
{
return(NodeId.NewInstance(str, CommonNodeLabelsManager.WildcardPort));
}
}
protected override void ConfigureServlets()
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.conf.Set(YarnConfiguration
.NmLocalDirs, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.testRootDir.GetAbsolutePath());
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.conf.Set(YarnConfiguration
.NmLogDirs, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.testLogDir.GetAbsolutePath());
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.conf);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.dirsHandler
= healthChecker.GetDiskHandler();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.aclsManager
= new ApplicationACLsManager(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.conf);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.nmContext
= new NodeManager.NMContext(null, null, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.dirsHandler, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.aclsManager, null);
NodeId nodeId = NodeId.NewInstance("testhost.foo.com", 9999);
((NodeManager.NMContext)Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.nmContext).SetNodeId(nodeId);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.resourceView
= new _ResourceView_106();
// 15.5G in bytes
// 16G in bytes
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.nmWebApp =
new WebServer.NMWebApp(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.resourceView, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.aclsManager, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.dirsHandler);
this.Bind <JAXBContextResolver>();
this.Bind <NMWebServices>();
this.Bind <GenericExceptionHandler>();
this.Bind <Context>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.nmContext);
this.Bind <WebApp>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.nmWebApp);
this.Bind <ResourceView>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.resourceView);
this.Bind <ApplicationACLsManager>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.aclsManager);
this.Bind <LocalDirsHandlerService>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
.dirsHandler);
this.Serve("/*").With(typeof(GuiceContainer));
}
public virtual void TestWriteContainer()
{
RMContainer container = CreateRMContainer(ContainerId.NewContainerId(ApplicationAttemptId
.NewInstance(ApplicationId.NewInstance(0, 1), 1), 1));
writer.ContainerStarted(container);
ContainerHistoryData containerHD = null;
for (int i = 0; i < MaxRetries; ++i)
{
containerHD = store.GetContainer(ContainerId.NewContainerId(ApplicationAttemptId.
NewInstance(ApplicationId.NewInstance(0, 1), 1), 1));
if (containerHD != null)
{
break;
}
else
{
Sharpen.Thread.Sleep(100);
}
}
NUnit.Framework.Assert.IsNotNull(containerHD);
NUnit.Framework.Assert.AreEqual(NodeId.NewInstance("test host", -100), containerHD
.GetAssignedNode());
NUnit.Framework.Assert.AreEqual(Resource.NewInstance(-1, -1), containerHD.GetAllocatedResource
());
NUnit.Framework.Assert.AreEqual(Priority.Undefined, containerHD.GetPriority());
NUnit.Framework.Assert.AreEqual(0L, container.GetCreationTime());
writer.ContainerFinished(container);
for (int i_1 = 0; i_1 < MaxRetries; ++i_1)
{
containerHD = store.GetContainer(ContainerId.NewContainerId(ApplicationAttemptId.
NewInstance(ApplicationId.NewInstance(0, 1), 1), 1));
if (containerHD.GetContainerState() != null)
{
break;
}
else
{
Sharpen.Thread.Sleep(100);
}
}
NUnit.Framework.Assert.AreEqual("test diagnostics info", containerHD.GetDiagnosticsInfo
());
NUnit.Framework.Assert.AreEqual(-1, containerHD.GetContainerExitStatus());
NUnit.Framework.Assert.AreEqual(ContainerState.Complete, containerHD.GetContainerState
());
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerAllocationWithSingleUserLimits()
{
RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
mgr.Init(conf);
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
.NewInstance("h2", 0), ToSet("y")));
// inject node label manager
MockRM rm1 = new _MockRM_471(mgr, GetConfigurationWithDefaultQueueLabels(conf));
rm1.GetRMContext().SetNodeLabelManager(mgr);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("h1:1234", 8000);
// label = x
rm1.RegisterNode("h2:1234", 8000);
// label = y
MockNM nm3 = rm1.RegisterNode("h3:1234", 8000);
// label = <empty>
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// A has only 10% of x, so it can only allocate one container in label=empty
ContainerId containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
), 2);
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty);
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
// Cannot allocate 2nd label=empty container
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 3);
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
// A has default user limit = 100, so it can use all resource in label = x
// We can allocate floor(8000 / 1024) = 7 containers
for (int id = 3; id <= 8; id++)
{
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), id);
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x");
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
}
rm1.Close();
}
public virtual void TestGetContainerReport()
{
ContainerId containerId = ContainerId.NewContainerId(ApplicationAttemptId.NewInstance
(ApplicationId.NewInstance(0, 1), 1), 1);
ContainerReport container;
if (callerUGI == null)
{
container = historyManager.GetContainer(containerId);
}
else
{
try
{
container = callerUGI.DoAs(new _PrivilegedExceptionAction_302(this, containerId));
if (callerUGI != null && callerUGI.GetShortUserName().Equals("user3"))
{
// The exception is expected
NUnit.Framework.Assert.Fail();
}
}
catch (AuthorizationException e)
{
if (callerUGI != null && callerUGI.GetShortUserName().Equals("user3"))
{
// The exception is expected
return;
}
throw;
}
}
NUnit.Framework.Assert.IsNotNull(container);
NUnit.Framework.Assert.AreEqual(int.MaxValue + 1L, container.GetCreationTime());
NUnit.Framework.Assert.AreEqual(int.MaxValue + 2L, container.GetFinishTime());
NUnit.Framework.Assert.AreEqual(Resource.NewInstance(-1, -1), container.GetAllocatedResource
());
NUnit.Framework.Assert.AreEqual(NodeId.NewInstance("test host", 100), container.GetAssignedNode
());
NUnit.Framework.Assert.AreEqual(Priority.Undefined, container.GetPriority());
NUnit.Framework.Assert.AreEqual("test diagnostics info", container.GetDiagnosticsInfo
());
NUnit.Framework.Assert.AreEqual(ContainerState.Complete, container.GetContainerState
());
NUnit.Framework.Assert.AreEqual(-1, container.GetContainerExitStatus());
NUnit.Framework.Assert.AreEqual("http://0.0.0.0:8188/applicationhistory/logs/" +
"test host:100/container_0_0001_01_000001/" + "container_0_0001_01_000001/user1"
, container.GetLogUrl());
}
/// <exception cref="System.IO.IOException"/>
public virtual LogParams GetLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID
)
{
JobId jobId = TypeConverter.ToYarn(oldJobID);
GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest
>();
request.SetJobId(jobId);
JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest
), request)).GetJobReport();
if (EnumSet.Of(JobState.Succeeded, JobState.Failed, JobState.Killed, JobState.Error
).Contains(report.GetJobState()))
{
if (oldTaskAttemptID != null)
{
GetTaskAttemptReportRequest taRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
>();
taRequest.SetTaskAttemptId(TypeConverter.ToYarn(oldTaskAttemptID));
TaskAttemptReport taReport = ((GetTaskAttemptReportResponse)Invoke("getTaskAttemptReport"
, typeof(GetTaskAttemptReportRequest), taRequest)).GetTaskAttemptReport();
if (taReport.GetContainerId() == null || taReport.GetNodeManagerHost() == null)
{
throw new IOException("Unable to get log information for task: " + oldTaskAttemptID
);
}
return(new LogParams(taReport.GetContainerId().ToString(), taReport.GetContainerId
().GetApplicationAttemptId().GetApplicationId().ToString(), NodeId.NewInstance(taReport
.GetNodeManagerHost(), taReport.GetNodeManagerPort()).ToString(), report.GetUser
()));
}
else
{
if (report.GetAMInfos() == null || report.GetAMInfos().Count == 0)
{
throw new IOException("Unable to get log information for job: " + oldJobID);
}
AMInfo amInfo = report.GetAMInfos()[report.GetAMInfos().Count - 1];
return(new LogParams(amInfo.GetContainerId().ToString(), amInfo.GetAppAttemptId()
.GetApplicationId().ToString(), NodeId.NewInstance(amInfo.GetNodeManagerHost(),
amInfo.GetNodeManagerPort()).ToString(), report.GetUser()));
}
}
else
{
throw new IOException("Cannot get log path for a in-progress job");
}
}
public virtual void TestRPCResponseId()
{
string node = "localhost";
Resource capability = BuilderUtils.NewResource(1024, 1);
RegisterNodeManagerRequest request = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
>();
nodeId = NodeId.NewInstance(node, 1234);
request.SetNodeId(nodeId);
request.SetHttpPort(0);
request.SetResource(capability);
RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
>();
request1.SetNodeId(nodeId);
request1.SetHttpPort(0);
request1.SetResource(capability);
resourceTrackerService.RegisterNodeManager(request1);
NodeStatus nodeStatus = recordFactory.NewRecordInstance <NodeStatus>();
nodeStatus.SetNodeId(nodeId);
NodeHealthStatus nodeHealthStatus = recordFactory.NewRecordInstance <NodeHealthStatus
>();
nodeHealthStatus.SetIsNodeHealthy(true);
nodeStatus.SetNodeHealthStatus(nodeHealthStatus);
NodeHeartbeatRequest nodeHeartBeatRequest = recordFactory.NewRecordInstance <NodeHeartbeatRequest
>();
nodeHeartBeatRequest.SetNodeStatus(nodeStatus);
nodeStatus.SetResponseId(0);
NodeHeartbeatResponse response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest
);
NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 1);
nodeStatus.SetResponseId(response.GetResponseId());
response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 2);
/* try calling with less response id */
response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 2);
nodeStatus.SetResponseId(0);
response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
NUnit.Framework.Assert.IsTrue(NodeAction.Resync.Equals(response.GetNodeAction()));
NUnit.Framework.Assert.AreEqual("Too far behind rm response id:2 nm response id:0"
, response.GetDiagnosticsMessage());
}