public virtual void TestApplicationAttemptId()
{
ApplicationAttemptId a1 = CreateAppAttemptId(10l, 1, 1);
ApplicationAttemptId a2 = CreateAppAttemptId(10l, 1, 2);
ApplicationAttemptId a3 = CreateAppAttemptId(10l, 2, 1);
ApplicationAttemptId a4 = CreateAppAttemptId(8l, 1, 4);
ApplicationAttemptId a5 = CreateAppAttemptId(10l, 1, 1);
NUnit.Framework.Assert.IsTrue(a1.Equals(a5));
NUnit.Framework.Assert.IsFalse(a1.Equals(a2));
NUnit.Framework.Assert.IsFalse(a1.Equals(a3));
NUnit.Framework.Assert.IsFalse(a1.Equals(a4));
NUnit.Framework.Assert.IsTrue(a1.CompareTo(a5) == 0);
NUnit.Framework.Assert.IsTrue(a1.CompareTo(a2) < 0);
NUnit.Framework.Assert.IsTrue(a1.CompareTo(a3) < 0);
NUnit.Framework.Assert.IsTrue(a1.CompareTo(a4) > 0);
NUnit.Framework.Assert.IsTrue(a1.GetHashCode() == a5.GetHashCode());
NUnit.Framework.Assert.IsFalse(a1.GetHashCode() == a2.GetHashCode());
NUnit.Framework.Assert.IsFalse(a1.GetHashCode() == a3.GetHashCode());
NUnit.Framework.Assert.IsFalse(a1.GetHashCode() == a4.GetHashCode());
long ts = Runtime.CurrentTimeMillis();
ApplicationAttemptId a6 = CreateAppAttemptId(ts, 543627, 33492611);
NUnit.Framework.Assert.AreEqual("appattempt_10_0001_000001", a1.ToString());
NUnit.Framework.Assert.AreEqual("appattempt_" + ts + "_543627_33492611", a6.ToString
());
}
/// <summary>
/// It will validate to make sure all the containers belong to correct
/// application attempt id.
/// </summary>
/// <remarks>
/// It will validate to make sure all the containers belong to correct
/// application attempt id. If not then it will throw
/// <see cref="Org.Apache.Hadoop.Yarn.Exceptions.InvalidContainerReleaseException"/>
/// </remarks>
/// <param name="containerReleaseList">containers to be released as requested by application master.
/// </param>
/// <param name="appAttemptId">Application attempt Id</param>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.InvalidContainerReleaseException
/// "/>
public static void ValidateContainerReleaseRequest(IList <ContainerId> containerReleaseList
, ApplicationAttemptId appAttemptId)
{
foreach (ContainerId cId in containerReleaseList)
{
if (!appAttemptId.Equals(cId.GetApplicationAttemptId()))
{
throw new InvalidContainerReleaseException("Cannot release container : " + cId.ToString
() + " not belonging to this application attempt : " + appAttemptId);
}
}
}
/// <exception cref="System.IO.IOException"/>
private ApplicationAttemptStateData CreateAttemptState(string itemName, byte[] data
)
{
ApplicationAttemptId attemptId = ConverterUtils.ToApplicationAttemptId(itemName);
ApplicationAttemptStateDataPBImpl attemptState = new ApplicationAttemptStateDataPBImpl
(YarnServerResourceManagerRecoveryProtos.ApplicationAttemptStateDataProto.ParseFrom
(data));
if (!attemptId.Equals(attemptState.GetAttemptId()))
{
throw new YarnRuntimeException("The database entry for " + attemptId + " contains data for "
+ attemptState.GetAttemptId());
}
return(attemptState);
}
public override void UnreserveResource(SchedulerApplicationAttempt application)
{
lock (this)
{
// Cannot unreserve for wrong application...
ApplicationAttemptId reservedApplication = GetReservedContainer().GetContainer().
GetId().GetApplicationAttemptId();
if (!reservedApplication.Equals(application.GetApplicationAttemptId()))
{
throw new InvalidOperationException("Trying to unreserve " + " for application "
+ application.GetApplicationId() + " when currently reserved " + " for application "
+ reservedApplication.GetApplicationId() + " on node " + this);
}
SetReservedContainer(null);
this.reservedAppSchedulable = null;
}
}
public override void UnreserveResource(SchedulerApplicationAttempt application)
{
lock (this)
{
// adding NP checks as this can now be called for preemption
if (GetReservedContainer() != null && GetReservedContainer().GetContainer() != null &&
GetReservedContainer().GetContainer().GetId() != null && GetReservedContainer
().GetContainer().GetId().GetApplicationAttemptId() != null)
{
// Cannot unreserve for wrong application...
ApplicationAttemptId reservedApplication = GetReservedContainer().GetContainer().
GetId().GetApplicationAttemptId();
if (!reservedApplication.Equals(application.GetApplicationAttemptId()))
{
throw new InvalidOperationException("Trying to unreserve " + " for application "
+ application.GetApplicationAttemptId() + " when currently reserved " + " for application "
+ reservedApplication.GetApplicationId() + " on node " + this);
}
}
SetReservedContainer(null);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAMRestartWithExistingContainers()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 2);
MockRM rm1 = new MockRM(conf);
rm1.Start();
RMApp app1 = rm1.SubmitApp(200, "name", "user", new Dictionary <ApplicationAccessType
, string>(), false, "default", -1, null, "MAPREDUCE", false, true);
MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockNM nm2 = new MockNM("127.0.0.1:2351", 4089, rm1.GetResourceTrackerService());
nm2.RegisterNode();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
int NumContainers = 3;
// allocate NUM_CONTAINERS containers
am1.Allocate("127.0.0.1", 1024, NumContainers, new AList <ContainerId>());
nm1.NodeHeartbeat(true);
// wait for containers to be allocated.
IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
while (containers.Count != NumContainers)
{
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, am1.Allocate(new AList <ResourceRequest>(),
new AList <ContainerId>()).GetAllocatedContainers());
Sharpen.Thread.Sleep(200);
}
// launch the 2nd container, for testing running container transferred.
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 2, ContainerState.Running);
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Running);
// launch the 3rd container, for testing container allocated by previous
// attempt is completed by the next new attempt/
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 3, ContainerState.Running);
ContainerId containerId3 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 3);
rm1.WaitForState(nm1, containerId3, RMContainerState.Running);
// 4th container still in AQUIRED state. for testing Acquired container is
// always killed.
ContainerId containerId4 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 4);
rm1.WaitForState(nm1, containerId4, RMContainerState.Acquired);
// 5th container is in Allocated state. for testing allocated container is
// always killed.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
nm1.NodeHeartbeat(true);
ContainerId containerId5 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 5);
rm1.WaitForContainerAllocated(nm1, containerId5);
rm1.WaitForState(nm1, containerId5, RMContainerState.Allocated);
// 6th container is in Reserved state.
am1.Allocate("127.0.0.1", 6000, 1, new AList <ContainerId>());
ContainerId containerId6 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 6);
nm1.NodeHeartbeat(true);
SchedulerApplicationAttempt schedulerAttempt = ((AbstractYarnScheduler)rm1.GetResourceScheduler
()).GetCurrentAttemptForContainer(containerId6);
while (schedulerAttempt.GetReservedContainers().IsEmpty())
{
System.Console.Out.WriteLine("Waiting for container " + containerId6 + " to be reserved."
);
nm1.NodeHeartbeat(true);
Sharpen.Thread.Sleep(200);
}
// assert containerId6 is reserved.
NUnit.Framework.Assert.AreEqual(containerId6, schedulerAttempt.GetReservedContainers
()[0].GetContainerId());
// fail the AM by sending CONTAINER_FINISHED event without registering.
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
am1.WaitForState(RMAppAttemptState.Failed);
// wait for some time. previous AM's running containers should still remain
// in scheduler even though am failed
Sharpen.Thread.Sleep(3000);
rm1.WaitForState(nm1, containerId2, RMContainerState.Running);
// acquired/allocated containers are cleaned up.
NUnit.Framework.Assert.IsNull(rm1.GetResourceScheduler().GetRMContainer(containerId4
));
NUnit.Framework.Assert.IsNull(rm1.GetResourceScheduler().GetRMContainer(containerId5
));
// wait for app to start a new attempt.
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// assert this is a new AM.
ApplicationAttemptId newAttemptId = app1.GetCurrentAppAttempt().GetAppAttemptId();
NUnit.Framework.Assert.IsFalse(newAttemptId.Equals(am1.GetApplicationAttemptId())
);
// launch the new AM
RMAppAttempt attempt2 = app1.GetCurrentAppAttempt();
nm1.NodeHeartbeat(true);
MockAM am2 = rm1.SendAMLaunched(attempt2.GetAppAttemptId());
RegisterApplicationMasterResponse registerResponse = am2.RegisterAppAttempt();
// Assert two containers are running: container2 and container3;
NUnit.Framework.Assert.AreEqual(2, registerResponse.GetContainersFromPreviousAttempts
().Count);
bool containerId2Exists = false;
bool containerId3Exists = false;
foreach (Container container in registerResponse.GetContainersFromPreviousAttempts
())
{
if (container.GetId().Equals(containerId2))
{
containerId2Exists = true;
}
if (container.GetId().Equals(containerId3))
{
containerId3Exists = true;
}
}
NUnit.Framework.Assert.IsTrue(containerId2Exists && containerId3Exists);
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Running);
// complete container by sending the container complete event which has earlier
// attempt's attemptId
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 3, ContainerState.Complete);
// Even though the completed container containerId3 event was sent to the
// earlier failed attempt, new RMAppAttempt can also capture this container
// info.
// completed containerId4 is also transferred to the new attempt.
RMAppAttempt newAttempt = app1.GetRMAppAttempt(am2.GetApplicationAttemptId());
// 4 containers finished, acquired/allocated/reserved/completed.
WaitForContainersToFinish(4, newAttempt);
bool container3Exists = false;
bool container4Exists = false;
bool container5Exists = false;
bool container6Exists = false;
foreach (ContainerStatus status in newAttempt.GetJustFinishedContainers())
{
if (status.GetContainerId().Equals(containerId3))
{
// containerId3 is the container ran by previous attempt but finished by the
// new attempt.
container3Exists = true;
}
if (status.GetContainerId().Equals(containerId4))
{
// containerId4 is the Acquired Container killed by the previous attempt,
// it's now inside new attempt's finished container list.
container4Exists = true;
}
if (status.GetContainerId().Equals(containerId5))
{
// containerId5 is the Allocated container killed by previous failed attempt.
container5Exists = true;
}
if (status.GetContainerId().Equals(containerId6))
{
// containerId6 is the reserved container killed by previous failed attempt.
container6Exists = true;
}
}
NUnit.Framework.Assert.IsTrue(container3Exists && container4Exists && container5Exists &&
container6Exists);
// New SchedulerApplicationAttempt also has the containers info.
rm1.WaitForState(nm1, containerId2, RMContainerState.Running);
// record the scheduler attempt for testing.
SchedulerApplicationAttempt schedulerNewAttempt = ((AbstractYarnScheduler)rm1.GetResourceScheduler
()).GetCurrentAttemptForContainer(containerId2);
// finish this application
MockRM.FinishAMAndVerifyAppState(app1, rm1, nm1, am2);
// the 2nd attempt released the 1st attempt's running container, when the
// 2nd attempt finishes.
NUnit.Framework.Assert.IsFalse(schedulerNewAttempt.GetLiveContainers().Contains(containerId2
));
// all 4 normal containers finished.
System.Console.Out.WriteLine("New attempt's just finished containers: " + newAttempt
.GetJustFinishedContainers());
WaitForContainersToFinish(5, newAttempt);
rm1.Stop();
}
