public virtual void TestPbRecordFactory()
{
RecordFactory pbRecordFactory = RecordFactoryPBImpl.Get();
try
{
AllocateResponse response = pbRecordFactory.NewRecordInstance <AllocateResponse>();
NUnit.Framework.Assert.AreEqual(typeof(AllocateResponsePBImpl), response.GetType(
));
}
catch (YarnRuntimeException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Failed to crete record");
}
try
{
AllocateRequest response = pbRecordFactory.NewRecordInstance <AllocateRequest>();
NUnit.Framework.Assert.AreEqual(typeof(AllocateRequestPBImpl), response.GetType()
);
}
catch (YarnRuntimeException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Failed to crete record");
}
}
public virtual void SetAllocateResponse(AllocateResponse response)
{
lock (this)
{
this.response = response;
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor()
{
Configuration conf = new Configuration();
TestAMRMClientAsync.TestCallbackHandler2 callbackHandler = new TestAMRMClientAsync.TestCallbackHandler2
(this);
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
IList <ContainerStatus> completed = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
(0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
AllocateResponse response = CreateAllocateResponse(completed, new AList <Container
>(), null);
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response
);
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 20, callbackHandler);
callbackHandler.asynClient = asyncClient;
asyncClient.Init(conf);
asyncClient.Start();
Supplier <bool> checker = new _Supplier_320(callbackHandler);
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
asyncClient.WaitFor(checker);
NUnit.Framework.Assert.IsTrue(checker.Get());
}
public IActionResult Allocate([FromBody] AllocateRequest spec)
{
if (spec == null)
{
return(BadRequest($"{nameof(spec)} can not be null"));
}
if (spec.Offers == null || spec.Offers.Length == 0)
{
return(BadRequest($"{nameof(spec.Offers)} can not be empty"));
}
if (spec.Threshold <= 0)
{
return(BadRequest($"{nameof(spec.Threshold)} must be greater than 0"));
}
if (spec.BusinessSeats < 0)
{
return(BadRequest($"{nameof(spec.BusinessSeats)} must be greater than or equal to 0"));
}
if (spec.EconomySeats < 0)
{
return(BadRequest($"{nameof(spec.EconomySeats)} must be greater than or equal to 0"));
}
var allocateResult = _allocationService.Allocate(spec.ToSpec());
return(Ok(AllocateResponse.From(allocateResult)));
}
/// <exception cref="System.IO.IOException"/>
public AllocateResponse Allocate(AllocateRequest request)
{
AllocateResponse response = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateResponse
>();
IList <ResourceRequest> askList = request.GetAskList();
IList <Container> containers = new AList <Container>();
foreach (ResourceRequest req in askList)
{
if (!ResourceRequest.IsAnyLocation(req.GetResourceName()))
{
continue;
}
int numContainers = req.GetNumContainers();
for (int i = 0; i < numContainers; i++)
{
ContainerId containerId = ContainerId.NewContainerId(this._enclosing.GetContext()
.GetApplicationAttemptId(), request.GetResponseId() + i);
containers.AddItem(Container.NewInstance(containerId, NodeId.NewInstance("host" +
containerId.GetContainerId(), 2345), "host" + containerId.GetContainerId() + ":5678"
, req.GetCapability(), req.GetPriority(), null));
}
}
response.SetAllocatedContainers(containers);
response.SetResponseId(request.GetResponseId() + 1);
response.SetNumClusterNodes(350);
return(response);
}
/// <exception cref="System.Exception"/>
public virtual AllocateResponse Schedule()
{
AllocateResponse response = Allocate(requests, releases);
requests.Clear();
releases.Clear();
return(response);
}
private AllocateResponse CreateAllocateResponse(IList <ContainerStatus> completed,
IList <Container> allocated, IList <NMToken> nmTokens)
{
AllocateResponse response = AllocateResponse.NewInstance(0, completed, allocated,
new AList <NodeReport>(), null, null, 1, null, nmTokens);
return(response);
}
/// <exception cref="System.Exception"/>
protected internal override void Heartbeat()
{
lock (this)
{
AllocateRequest allocateRequest = AllocateRequest.NewInstance(this.lastResponseID
, base.GetApplicationProgress(), new AList <ResourceRequest>(), new AList <ContainerId
>(), null);
AllocateResponse allocateResponse = null;
try
{
allocateResponse = scheduler.Allocate(allocateRequest);
// Reset retry count if no exception occurred.
retrystartTime = Runtime.CurrentTimeMillis();
}
catch (ApplicationAttemptNotFoundException e)
{
Log.Info("Event from RM: shutting down Application Master");
// This can happen if the RM has been restarted. If it is in that state,
// this application must clean itself up.
eventHandler.Handle(new JobEvent(this.GetJob().GetID(), JobEventType.JobAmReboot)
);
throw new YarnRuntimeException("Resource Manager doesn't recognize AttemptId: " +
this.GetContext().GetApplicationID(), e);
}
catch (ApplicationMasterNotRegisteredException)
{
Log.Info("ApplicationMaster is out of sync with ResourceManager," + " hence resync and send outstanding requests."
);
this.lastResponseID = 0;
Register();
}
catch (Exception e)
{
// This can happen when the connection to the RM has gone down. Keep
// re-trying until the retryInterval has expired.
if (Runtime.CurrentTimeMillis() - retrystartTime >= retryInterval)
{
Log.Error("Could not contact RM after " + retryInterval + " milliseconds.");
eventHandler.Handle(new JobEvent(this.GetJob().GetID(), JobEventType.InternalError
));
throw new YarnRuntimeException("Could not contact RM after " + retryInterval + " milliseconds."
);
}
// Throw this up to the caller, which may decide to ignore it and
// continue to attempt to contact the RM.
throw;
}
if (allocateResponse != null)
{
this.lastResponseID = allocateResponse.GetResponseId();
Token token = allocateResponse.GetAMRMToken();
if (token != null)
{
UpdateAMRMToken(token);
}
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidContainerReleaseRequest()
{
MockRM rm = new MockRM(conf);
try
{
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(1024);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
AllocateResponse alloc1Response = am1.Schedule();
// send the request
// kick the scheduler
nm1.NodeHeartbeat(true);
while (alloc1Response.GetAllocatedContainers().Count < 1)
{
Log.Info("Waiting for containers to be created for app 1...");
Sharpen.Thread.Sleep(1000);
alloc1Response = am1.Schedule();
}
NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0);
RMApp app2 = rm.SubmitApp(1024);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId());
am2.RegisterAppAttempt();
// Now trying to release container allocated for app1 -> appAttempt1.
ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId();
am2.AddContainerToBeReleased(cId);
try
{
am2.Schedule();
NUnit.Framework.Assert.Fail("Exception was expected!!");
}
catch (InvalidContainerReleaseException e)
{
StringBuilder sb = new StringBuilder("Cannot release container : ");
sb.Append(cId.ToString());
sb.Append(" not belonging to this application attempt : ");
sb.Append(attempt2.GetAppAttemptId().ToString());
NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString()));
}
}
finally
{
if (rm != null)
{
rm.Stop();
}
}
}
public override void Run()
{
while (true)
{
AllocateResponse response = null;
// synchronization ensures we don't send heartbeats after unregistering
lock (this._enclosing.unregisterHeartbeatLock)
{
if (!this._enclosing.keepRunning)
{
return;
}
try
{
response = this._enclosing.client.Allocate(this._enclosing.progress);
}
catch (ApplicationAttemptNotFoundException)
{
this._enclosing.handler.OnShutdownRequest();
AMRMClientAsyncImpl.Log.Info("Shutdown requested. Stopping callback.");
return;
}
catch (Exception ex)
{
AMRMClientAsyncImpl.Log.Error("Exception on heartbeat", ex);
this._enclosing.savedException = ex;
// interrupt handler thread in case it waiting on the queue
this._enclosing.handlerThread.Interrupt();
return;
}
if (response != null)
{
while (true)
{
try
{
this._enclosing.responseQueue.Put(response);
break;
}
catch (Exception ex)
{
AMRMClientAsyncImpl.Log.Debug("Interrupted while waiting to put on response queue"
, ex);
}
}
}
}
try
{
Sharpen.Thread.Sleep(this._enclosing.heartbeatIntervalMs.Get());
}
catch (Exception ex)
{
AMRMClientAsyncImpl.Log.Debug("Heartbeater interrupted", ex);
}
}
}
// To avoid using cached client
/// <exception cref="System.Exception"/>
public virtual void TestAMRMMasterKeysUpdate()
{
AtomicReference <AMRMTokenSecretManager> spySecretMgrRef = new AtomicReference <AMRMTokenSecretManager
>();
MockRM rm = new _MockRM_349(this, spySecretMgrRef, conf);
// Skip the login.
rm.Start();
MockNM nm = rm.RegisterNode("127.0.0.1:1234", 8000);
RMApp app = rm.SubmitApp(200);
MockAM am = MockRM.LaunchAndRegisterAM(app, rm, nm);
AMRMTokenSecretManager spySecretMgr = spySecretMgrRef.Get();
// Do allocate. Should not update AMRMToken
AllocateResponse response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<AllocateRequest>());
NUnit.Framework.Assert.IsNull(response.GetAMRMToken());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> oldToken = rm.GetRMContext
().GetRMApps()[app.GetApplicationId()].GetRMAppAttempt(am.GetApplicationAttemptId
()).GetAMRMToken();
// roll over the master key
// Do allocate again. the AM should get the latest AMRMToken
rm.GetRMContext().GetAMRMTokenSecretManager().RollMasterKey();
response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNotNull(response.GetAMRMToken());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = ConverterUtils
.ConvertFromYarn(response.GetAMRMToken(), new Text(response.GetAMRMToken().GetService
()));
NUnit.Framework.Assert.AreEqual(amrmToken.DecodeIdentifier().GetKeyId(), rm.GetRMContext
().GetAMRMTokenSecretManager().GetMasterKey().GetMasterKey().GetKeyId());
// Do allocate again with the same old token and verify the RM sends
// back the last generated token instead of generating it again.
Org.Mockito.Mockito.Reset(spySecretMgr);
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(am.GetApplicationAttemptId
().ToString(), new string[0]);
ugi.AddTokenIdentifier(oldToken.DecodeIdentifier());
response = am.DoAllocateAs(ugi, Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNotNull(response.GetAMRMToken());
Org.Mockito.Mockito.Verify(spySecretMgr, Org.Mockito.Mockito.Never()).CreateAndGetAMRMToken
(Matchers.IsA <ApplicationAttemptId>());
// Do allocate again with the updated token and verify we do not
// receive a new token to use.
response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNull(response.GetAMRMToken());
// Activate the next master key. Since there is new master key generated
// in AMRMTokenSecretManager. The AMRMToken will not get updated for AM
rm.GetRMContext().GetAMRMTokenSecretManager().ActivateNextMasterKey();
response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNull(response.GetAMRMToken());
rm.Stop();
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestAllocateOnHA()
{
AllocateRequest request = AllocateRequest.NewInstance(0, 50f, new AList <ResourceRequest
>(), new AList <ContainerId>(), ResourceBlacklistRequest.NewInstance(new AList <string
>(), new AList <string>()));
AllocateResponse response = amClient.Allocate(request);
NUnit.Framework.Assert.AreEqual(response, this.cluster.CreateFakeAllocateResponse
());
}
/// <exception cref="System.Exception"/>
public virtual AllocateResponse Allocate(AllocateRequest allocateRequest)
{
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(attemptId.ToString
());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = context.GetRMApps
()[attemptId.GetApplicationId()].GetRMAppAttempt(attemptId).GetAMRMToken();
ugi.AddTokenIdentifier(token.DecodeIdentifier());
lastResponse = DoAllocateAs(ugi, allocateRequest);
return(lastResponse);
}
public virtual void RegisterAppAttempt(ApplicationAttemptId attemptId)
{
AllocateResponse response = recordFactory.NewRecordInstance <AllocateResponse>();
// set response id to -1 before application master for the following
// attemptID get registered
response.SetResponseId(-1);
Log.Info("Registering app attempt : " + attemptId);
responseMap[attemptId] = new ApplicationMasterService.AllocateResponseLock(response
);
rmContext.GetNMTokenSecretManager().RegisterApplicationAttempt(attemptId);
}
// Test even if AM container is allocated with containerId not equal to 1, the
// following allocate requests from AM should be able to retrieve the
// corresponding NM Token.
/// <exception cref="System.Exception"/>
public virtual void TestNMTokenSentForNormalContainer()
{
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).GetCanonicalName
());
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
// Call getNewContainerId to increase container Id so that the AM container
// Id doesn't equal to one.
CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();
cs.GetApplicationAttempt(attempt.GetAppAttemptId()).GetNewContainerId();
// kick the scheduling
nm1.NodeHeartbeat(true);
MockAM am = MockRM.LaunchAM(app, rm, nm1);
// am container Id not equal to 1.
NUnit.Framework.Assert.IsTrue(attempt.GetMasterContainer().GetId().GetContainerId
() != 1);
// NMSecretManager doesn't record the node on which the am is allocated.
NUnit.Framework.Assert.IsFalse(rm.GetRMContext().GetNMTokenSecretManager().IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm1.GetNodeId()));
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
int NumContainers = 1;
IList <Container> containers = new AList <Container>();
// nmTokens keeps track of all the nmTokens issued in the allocate call.
IList <NMToken> expectedNMTokens = new AList <NMToken>();
// am1 allocate 1 container on nm1.
while (true)
{
AllocateResponse response = am.Allocate("127.0.0.1", 2000, NumContainers, new AList
<ContainerId>());
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers());
Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens());
if (containers.Count == NumContainers)
{
break;
}
Sharpen.Thread.Sleep(200);
System.Console.Out.WriteLine("Waiting for container to be allocated.");
}
NodeId nodeId = expectedNMTokens[0].GetNodeId();
// NMToken is sent for the allocated container.
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), nodeId);
}
/// <exception cref="System.Exception"/>
public virtual void TestAppCleanupWhenRMRestartedBeforeAppFinished()
{
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
// start RM
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 1024, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockNM nm2 = new MockNM("127.0.0.1:5678", 1024, rm1.GetResourceTrackerService());
nm2.RegisterNode();
// create app and launch the AM
RMApp app0 = rm1.SubmitApp(200);
MockAM am0 = LaunchAM(app0, rm1, nm1);
// alloc another container on nm2
AllocateResponse allocResponse = am0.Allocate(Arrays.AsList(ResourceRequest.NewInstance
(Priority.NewInstance(1), "*", Resource.NewInstance(1024, 0), 1)), null);
while (null == allocResponse.GetAllocatedContainers() || allocResponse.GetAllocatedContainers
().IsEmpty())
{
nm2.NodeHeartbeat(true);
allocResponse = am0.Allocate(null, null);
Sharpen.Thread.Sleep(1000);
}
// start new RM
MockRM rm2 = new MockRM(conf, memStore);
rm2.Start();
// nm1/nm2 register to rm2, and do a heartbeat
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm1.RegisterNode(Arrays.AsList(NMContainerStatus.NewInstance(ContainerId.NewContainerId
(am0.GetApplicationAttemptId(), 1), ContainerState.Complete, Resource.NewInstance
(1024, 1), string.Empty, 0, Priority.NewInstance(0), 1234)), Arrays.AsList(app0.
GetApplicationId()));
nm2.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm2.RegisterNode(Arrays.AsList(app0.GetApplicationId()));
// assert app state has been saved.
rm2.WaitForState(app0.GetApplicationId(), RMAppState.Failed);
// wait for application cleanup message received on NM1
WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId());
// wait for application cleanup message received on NM2
WaitForAppCleanupMessageRecved(nm2, app0.GetApplicationId());
rm1.Stop();
rm2.Stop();
}
public virtual void TestAllocateResponseWithoutIncDecContainers()
{
AllocateResponse r = AllocateResponse.NewInstance(3, new AList <ContainerStatus>()
, new AList <Container>(), new AList <NodeReport>(), null, AMCommand.AmResync, 3,
null, new AList <NMToken>(), null, null);
// serde
YarnServiceProtos.AllocateResponseProto p = ((AllocateResponsePBImpl)r).GetProto(
);
r = new AllocateResponsePBImpl(p);
// check value
NUnit.Framework.Assert.AreEqual(0, r.GetIncreasedContainers().Count);
NUnit.Framework.Assert.AreEqual(0, r.GetDecreasedContainers().Count);
}
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
internal virtual void RunCallBackThrowOutException(TestAMRMClientAsync.TestCallbackHandler2
callbackHandler)
{
Configuration conf = new Configuration();
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
IList <ContainerStatus> completed = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
(0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
AllocateResponse response = CreateAllocateResponse(completed, new AList <Container
>(), null);
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response
);
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 20, callbackHandler);
callbackHandler.asynClient = asyncClient;
callbackHandler.throwOutException = true;
asyncClient.Init(conf);
asyncClient.Start();
// call register and wait for error callback and stop
lock (callbackHandler.notifier)
{
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
while (callbackHandler.notify == false)
{
try
{
Sharpen.Runtime.Wait(callbackHandler.notifier);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
}
// verify error invoked
Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(0)).GetProgress
();
Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(1)).OnError
(Matchers.Any <Exception>());
// sleep to wait for a few heartbeat calls that can trigger callbacks
Sharpen.Thread.Sleep(50);
// verify no more invocations after the first one.
// ie. callback thread has stopped
Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(0)).GetProgress
();
Org.Mockito.Mockito.Verify(callbackHandler, Org.Mockito.Mockito.Times(1)).OnError
(Matchers.Any <Exception>());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
private ICollection <Container> AllocateContainers(AMRMClientImpl <AMRMClient.ContainerRequest
> rmClient, int num)
{
// setup container request
Resource capability = Resource.NewInstance(1024, 0);
Priority priority = Priority.NewInstance(0);
string node = nodeReports[0].GetNodeId().GetHost();
string rack = nodeReports[0].GetRackName();
string[] nodes = new string[] { node };
string[] racks = new string[] { rack };
for (int i = 0; i < num; ++i)
{
rmClient.AddContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks
, priority));
}
int containersRequestedAny = rmClient.remoteRequestsTable[priority][ResourceRequest
.Any][capability].remoteRequest.GetNumContainers();
// RM should allocate container within 2 calls to allocate()
int allocatedContainerCount = 0;
int iterationsLeft = 2;
ICollection <Container> containers = new TreeSet <Container>();
while (allocatedContainerCount < containersRequestedAny && iterationsLeft > 0)
{
AllocateResponse allocResponse = rmClient.Allocate(0.1f);
allocatedContainerCount += allocResponse.GetAllocatedContainers().Count;
foreach (Container container in allocResponse.GetAllocatedContainers())
{
containers.AddItem(container);
}
if (!allocResponse.GetNMTokens().IsEmpty())
{
foreach (NMToken token in allocResponse.GetNMTokens())
{
rmClient.GetNMTokenCache().SetToken(token.GetNodeId().ToString(), token.GetToken(
));
}
}
if (allocatedContainerCount < containersRequestedAny)
{
// sleep to let NM's heartbeat to RM and trigger allocations
Sleep(1000);
}
--iterationsLeft;
}
return(containers);
}
public static AllocateResponse NewAllocateResponse(int responseId, IList <ContainerStatus
> completedContainers, IList <Container> allocatedContainers, IList <NodeReport> updatedNodes
, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage
preempt)
{
AllocateResponse response = recordFactory.NewRecordInstance <AllocateResponse>();
response.SetNumClusterNodes(numClusterNodes);
response.SetResponseId(responseId);
response.SetCompletedContainersStatuses(completedContainers);
response.SetAllocatedContainers(allocatedContainers);
response.SetUpdatedNodes(updatedNodes);
response.SetAvailableResources(availResources);
response.SetAMCommand(command);
response.SetPreemptionMessage(preempt);
return(response);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual AllocateResponse Allocate(AllocateRequest request)
{
NUnit.Framework.Assert.AreEqual("response ID mismatch", responseId, request.GetResponseId
());
++responseId;
Org.Apache.Hadoop.Yarn.Api.Records.Token yarnToken = null;
if (amToken != null)
{
yarnToken = Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance(amToken.GetIdentifier
(), amToken.GetKind().ToString(), amToken.GetPassword(), amToken.GetService().ToString
());
}
return(AllocateResponse.NewInstance(responseId, Collections.EmptyList <ContainerStatus
>(), Collections.EmptyList <Container>(), Collections.EmptyList <NodeReport>(), Resources
.None(), null, 1, null, Collections.EmptyList <NMToken>(), yarnToken, Collections
.EmptyList <ContainerResourceIncrease>(), Collections.EmptyList <ContainerResourceDecrease
>()));
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual YarnServiceProtos.AllocateResponseProto Allocate(RpcController arg0
, YarnServiceProtos.AllocateRequestProto proto)
{
AllocateRequestPBImpl request = new AllocateRequestPBImpl(proto);
try
{
AllocateResponse response = real.Allocate(request);
return(((AllocateResponsePBImpl)response).GetProto());
}
catch (YarnException e)
{
throw new ServiceException(e);
}
catch (IOException e)
{
throw new ServiceException(e);
}
}
public virtual void TestAllocateResponseWithIncDecContainers()
{
IList <ContainerResourceIncrease> incContainers = new AList <ContainerResourceIncrease
>();
IList <ContainerResourceDecrease> decContainers = new AList <ContainerResourceDecrease
>();
for (int i = 0; i < 3; i++)
{
incContainers.AddItem(ContainerResourceIncrease.NewInstance(null, Resource.NewInstance
(1024, i), null));
}
for (int i_1 = 0; i_1 < 5; i_1++)
{
decContainers.AddItem(ContainerResourceDecrease.NewInstance(null, Resource.NewInstance
(1024, i_1)));
}
AllocateResponse r = AllocateResponse.NewInstance(3, new AList <ContainerStatus>()
, new AList <Container>(), new AList <NodeReport>(), null, AMCommand.AmResync, 3,
null, new AList <NMToken>(), incContainers, decContainers);
// serde
YarnServiceProtos.AllocateResponseProto p = ((AllocateResponsePBImpl)r).GetProto(
);
r = new AllocateResponsePBImpl(p);
// check value
NUnit.Framework.Assert.AreEqual(incContainers.Count, r.GetIncreasedContainers().Count
);
NUnit.Framework.Assert.AreEqual(decContainers.Count, r.GetDecreasedContainers().Count
);
for (int i_2 = 0; i_2 < incContainers.Count; i_2++)
{
NUnit.Framework.Assert.AreEqual(i_2, r.GetIncreasedContainers()[i_2].GetCapability
().GetVirtualCores());
}
for (int i_3 = 0; i_3 < decContainers.Count; i_3++)
{
NUnit.Framework.Assert.AreEqual(i_3, r.GetDecreasedContainers()[i_3].GetCapability
().GetVirtualCores());
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
protected internal virtual AllocateResponse MakeRemoteRequest()
{
ApplyRequestLimits();
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
new AList <string>(blacklistAdditions), new AList <string>(blacklistRemovals));
AllocateRequest allocateRequest = AllocateRequest.NewInstance(lastResponseID, base
.GetApplicationProgress(), new AList <ResourceRequest>(ask), new AList <ContainerId
>(release), blacklistRequest);
AllocateResponse allocateResponse = scheduler.Allocate(allocateRequest);
lastResponseID = allocateResponse.GetResponseId();
availableResources = allocateResponse.GetAvailableResources();
lastClusterNmCount = clusterNmCount;
clusterNmCount = allocateResponse.GetNumClusterNodes();
int numCompletedContainers = allocateResponse.GetCompletedContainersStatuses().Count;
if (ask.Count > 0 || release.Count > 0)
{
Log.Info("getResources() for " + applicationId + ":" + " ask=" + ask.Count + " release= "
+ release.Count + " newContainers=" + allocateResponse.GetAllocatedContainers()
.Count + " finishedContainers=" + numCompletedContainers + " resourcelimit=" + availableResources
+ " knownNMs=" + clusterNmCount);
}
ask.Clear();
release.Clear();
if (numCompletedContainers > 0)
{
// re-send limited requests when a container completes to trigger asking
// for more containers
Sharpen.Collections.AddAll(requestLimitsToUpdate, requestLimits.Keys);
}
if (blacklistAdditions.Count > 0 || blacklistRemovals.Count > 0)
{
Log.Info("Update the blacklist for " + applicationId + ": blacklistAdditions=" +
blacklistAdditions.Count + " blacklistRemovals=" + blacklistRemovals.Count);
}
blacklistAdditions.Clear();
blacklistRemovals.Clear();
return(allocateResponse);
}
/// <exception cref="System.Exception"/>
public virtual void TestRMIdentifierOnContainerAllocation()
{
MockRM rm = new MockRM(conf);
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(2048);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
AllocateResponse alloc1Response = am1.Schedule();
// send the request
// kick the scheduler
nm1.NodeHeartbeat(true);
while (alloc1Response.GetAllocatedContainers().Count < 1)
{
Log.Info("Waiting for containers to be created for app 1...");
Sharpen.Thread.Sleep(1000);
alloc1Response = am1.Schedule();
}
// assert RMIdentifer is set properly in allocated containers
Container allocatedContainer = alloc1Response.GetAllocatedContainers()[0];
ContainerTokenIdentifier tokenId = BuilderUtils.NewContainerTokenIdentifier(allocatedContainer
.GetContainerToken());
NUnit.Framework.Assert.AreEqual(MockRM.GetClusterTimeStamp(), tokenId.GetRMIdentifier
());
rm.Stop();
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestCallAMRMClientAsyncStopFromCallbackHandler()
{
Configuration conf = new Configuration();
TestAMRMClientAsync.TestCallbackHandler2 callbackHandler = new TestAMRMClientAsync.TestCallbackHandler2
(this);
AMRMClient <AMRMClient.ContainerRequest> client = Org.Mockito.Mockito.Mock <AMRMClientImpl
>();
IList <ContainerStatus> completed = Arrays.AsList(ContainerStatus.NewInstance(NewContainerId
(0, 0, 0, 0), ContainerState.Complete, string.Empty, 0));
AllocateResponse response = CreateAllocateResponse(completed, new AList <Container
>(), null);
Org.Mockito.Mockito.When(client.Allocate(Matchers.AnyFloat())).ThenReturn(response
);
AMRMClientAsync <AMRMClient.ContainerRequest> asyncClient = AMRMClientAsync.CreateAMRMClientAsync
(client, 20, callbackHandler);
callbackHandler.asynClient = asyncClient;
asyncClient.Init(conf);
asyncClient.Start();
lock (callbackHandler.notifier)
{
asyncClient.RegisterApplicationMaster("localhost", 1234, null);
while (callbackHandler.notify == false)
{
try
{
Sharpen.Runtime.Wait(callbackHandler.notifier);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
}
}
public virtual void TestARRMResponseId()
{
MockNM nm1 = rm.RegisterNode("h1:1234", 5000);
RMApp app = rm.SubmitApp(2000);
// Trigger the scheduling so the AM gets 'launched'
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0F, null, null,
null);
AllocateResponse response = Allocate(attempt.GetAppAttemptId(), allocateRequest);
NUnit.Framework.Assert.AreEqual(1, response.GetResponseId());
NUnit.Framework.Assert.IsTrue(response.GetAMCommand() == null);
allocateRequest = AllocateRequest.NewInstance(response.GetResponseId(), 0F, null,
null, null);
response = Allocate(attempt.GetAppAttemptId(), allocateRequest);
NUnit.Framework.Assert.AreEqual(2, response.GetResponseId());
/* try resending */
response = Allocate(attempt.GetAppAttemptId(), allocateRequest);
NUnit.Framework.Assert.AreEqual(2, response.GetResponseId());
allocateRequest = AllocateRequest.NewInstance(0, 0F, null, null, null);
try
{
Allocate(attempt.GetAppAttemptId(), allocateRequest);
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException is InvalidApplicationMasterRequestException
);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestNMTokensRebindOnAMRestart()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 3);
MockRM rm1 = new MockRM(conf);
rm1.Start();
RMApp app1 = rm1.SubmitApp(200, "myname", "myuser", new Dictionary <ApplicationAccessType
, string>(), false, "default", -1, null, "MAPREDUCE", false, true);
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockNM nm2 = new MockNM("127.1.1.1:4321", 8000, rm1.GetResourceTrackerService());
nm2.RegisterNode();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
IList <Container> containers = new AList <Container>();
// nmTokens keeps track of all the nmTokens issued in the allocate call.
IList <NMToken> expectedNMTokens = new AList <NMToken>();
// am1 allocate 2 container on nm1.
// first container
while (true)
{
AllocateResponse response = am1.Allocate("127.0.0.1", 2000, 2, new AList <ContainerId
>());
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers());
Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens());
if (containers.Count == 2)
{
break;
}
Sharpen.Thread.Sleep(200);
System.Console.Out.WriteLine("Waiting for container to be allocated.");
}
// launch the container-2
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 2, ContainerState.Running);
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Running);
// launch the container-3
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 3, ContainerState.Running);
ContainerId containerId3 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 3);
rm1.WaitForState(nm1, containerId3, RMContainerState.Running);
// fail am1
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
am1.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// restart the am
MockAM am2 = MockRM.LaunchAM(app1, rm1, nm1);
RegisterApplicationMasterResponse registerResponse = am2.RegisterAppAttempt();
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Running);
// check am2 get the nm token from am1.
NUnit.Framework.Assert.AreEqual(expectedNMTokens, registerResponse.GetNMTokensFromPreviousAttempts
());
// am2 allocate 1 container on nm2
containers = new AList <Container>();
while (true)
{
AllocateResponse allocateResponse = am2.Allocate("127.1.1.1", 4000, 1, new AList <
ContainerId>());
nm2.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, allocateResponse.GetAllocatedContainers());
Sharpen.Collections.AddAll(expectedNMTokens, allocateResponse.GetNMTokens());
if (containers.Count == 1)
{
break;
}
Sharpen.Thread.Sleep(200);
System.Console.Out.WriteLine("Waiting for container to be allocated.");
}
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 2, ContainerState.Running);
ContainerId am2ContainerId2 = ContainerId.NewContainerId(am2.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, am2ContainerId2, RMContainerState.Running);
// fail am2.
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// restart am
MockAM am3 = MockRM.LaunchAM(app1, rm1, nm1);
registerResponse = am3.RegisterAppAttempt();
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Running);
// check am3 get the NM token from both am1 and am2;
IList <NMToken> transferredTokens = registerResponse.GetNMTokensFromPreviousAttempts
();
NUnit.Framework.Assert.AreEqual(2, transferredTokens.Count);
NUnit.Framework.Assert.IsTrue(transferredTokens.ContainsAll(expectedNMTokens));
rm1.Stop();
}
// The test verifies processing of NMContainerStatuses which are sent during
// NM registration.
// 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM
// 2. AM sends ResourceRequest for 1 container with memory 2048MB.
// 3. Verify for number of container allocated by RM
// 4. Verify Memory Usage by cluster, it should be 3072. AM memory + requested
// memory. 1024 + 2048=3072
// 5. Re-register NM by sending completed container status
// 6. Verify for Memory Used, it should be 1024
// 7. Send AM heatbeat to RM. Allocated response should contain completed
// container.
/// <exception cref="System.Exception"/>
public virtual void TestProcessingNMContainerStatusesOnNMRestart()
{
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
// 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
int nmMemory = 8192;
int amMemory = 1024;
int containerMemory = 2048;
MockNM nm1 = new MockNM("127.0.0.1:1234", nmMemory, rm1.GetResourceTrackerService
());
nm1.RegisterNode();
RMApp app0 = rm1.SubmitApp(amMemory);
MockAM am0 = MockRM.LaunchAndRegisterAM(app0, rm1, nm1);
// 2. AM sends ResourceRequest for 1 container with memory 2048MB.
int noOfContainers = 1;
IList <Container> allocateContainers = am0.AllocateAndWaitForContainers(noOfContainers
, containerMemory, nm1);
// 3. Verify for number of container allocated by RM
NUnit.Framework.Assert.AreEqual(noOfContainers, allocateContainers.Count);
Container container = allocateContainers[0];
nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), 1, ContainerState.Running);
nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), container.GetId().GetContainerId
(), ContainerState.Running);
rm1.WaitForState(app0.GetApplicationId(), RMAppState.Running);
// 4. Verify Memory Usage by cluster, it should be 3072. AM memory +
// requested memory. 1024 + 2048=3072
ResourceScheduler rs = rm1.GetRMContext().GetScheduler();
int allocatedMB = rs.GetRootQueueMetrics().GetAllocatedMB();
NUnit.Framework.Assert.AreEqual(amMemory + containerMemory, allocatedMB);
// 5. Re-register NM by sending completed container status
IList <NMContainerStatus> nMContainerStatusForApp = CreateNMContainerStatusForApp(
am0);
nm1.RegisterNode(nMContainerStatusForApp, Arrays.AsList(app0.GetApplicationId()));
WaitForClusterMemory(nm1, rs, amMemory);
// 6. Verify for Memory Used, it should be 1024
NUnit.Framework.Assert.AreEqual(amMemory, rs.GetRootQueueMetrics().GetAllocatedMB
());
// 7. Send AM heatbeat to RM. Allocated response should contain completed
// container
AllocateRequest req = AllocateRequest.NewInstance(0, 0F, new AList <ResourceRequest
>(), new AList <ContainerId>(), null);
AllocateResponse allocate = am0.Allocate(req);
IList <ContainerStatus> completedContainersStatuses = allocate.GetCompletedContainersStatuses
();
NUnit.Framework.Assert.AreEqual(noOfContainers, completedContainersStatuses.Count
);
// Application clean up should happen Cluster memory used is 0
nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), 1, ContainerState.Complete);
WaitForClusterMemory(nm1, rs, 0);
rm1.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestExcessReservationThanNodeManagerCapacity()
{
MockRM rm = new MockRM(conf);
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 2 * Gb, 4);
MockNM nm2 = rm.RegisterNode("127.0.0.1:2234", 3 * Gb, 4);
nm1.NodeHeartbeat(true);
nm2.NodeHeartbeat(true);
// wait..
int waitCount = 20;
int size = rm.GetRMContext().GetRMNodes().Count;
while ((size = rm.GetRMContext().GetRMNodes().Count) != 2 && waitCount-- > 0)
{
Log.Info("Waiting for node managers to register : " + size);
Sharpen.Thread.Sleep(100);
}
NUnit.Framework.Assert.AreEqual(2, rm.GetRMContext().GetRMNodes().Count);
// Submit an application
RMApp app1 = rm.SubmitApp(128);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
Log.Info("sending container requests ");
am1.AddRequests(new string[] { "*" }, 2 * Gb, 1, 1);
AllocateResponse alloc1Response = am1.Schedule();
// send the request
// kick the scheduler
nm1.NodeHeartbeat(true);
int waitCounter = 20;
Log.Info("heartbeating nm1");
while (alloc1Response.GetAllocatedContainers().Count < 1 && waitCounter-- > 0)
{
Log.Info("Waiting for containers to be created for app 1...");
Sharpen.Thread.Sleep(500);
alloc1Response = am1.Schedule();
}
Log.Info("received container : " + alloc1Response.GetAllocatedContainers().Count);
// No container should be allocated.
// Internally it should not been reserved.
NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count == 0);
Log.Info("heartbeating nm2");
waitCounter = 20;
nm2.NodeHeartbeat(true);
while (alloc1Response.GetAllocatedContainers().Count < 1 && waitCounter-- > 0)
{
Log.Info("Waiting for containers to be created for app 1...");
Sharpen.Thread.Sleep(500);
alloc1Response = am1.Schedule();
}
Log.Info("received container : " + alloc1Response.GetAllocatedContainers().Count);
NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count == 1);
rm.Stop();
}
