public virtual ReservationId GetNewReservationId()
{
writeLock.Lock();
try
{
ReservationId resId = ReservationId.NewInstance(ResourceManager.GetClusterTimeStamp
(), resCounter.IncrementAndGet());
Log.Info("Allocated new reservationId: " + resId);
return(resId);
}
finally
{
writeLock.Unlock();
}
}
public virtual void TestZeroAlloaction()
{
ReservationId reservationID = ReservationId.NewInstance(rand.NextLong(), rand.NextLong
());
int[] alloc = new int[] { };
long start = 0;
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length + 1, alloc.Length);
IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary
<ReservationInterval, ReservationRequest>();
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length + 1, allocations, resCalc, minAlloc
);
DoAssertions(rAllocation, reservationID, rDef, allocations, (int)start, alloc);
NUnit.Framework.Assert.IsFalse(rAllocation.ContainsGangs());
}
public virtual void TestBlocks()
{
ReservationId reservationID = ReservationId.NewInstance(rand.NextLong(), rand.NextLong
());
int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 };
int start = 100;
ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc
.Length + 1, alloc.Length);
IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation
(start, alloc, false, false);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID
, rDef, user, planName, start, start + alloc.Length + 1, allocations, resCalc, minAlloc
);
DoAssertions(rAllocation, reservationID, rDef, allocations, start, alloc);
NUnit.Framework.Assert.IsFalse(rAllocation.ContainsGangs());
for (int i = 0; i < alloc.Length; i++)
{
NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance
(1024 * (alloc[i]), (alloc[i])), rAllocation.GetResourcesAtTime(start + i));
}
}
public static ReservationId GetNewReservationId()
{
return(ReservationId.NewInstance(rand.NextLong(), rand.NextLong()));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/>
protected internal virtual void TestPlanFollower(bool isMove)
{
// Initialize plan based on move flag
plan = new InMemoryPlan(scheduler.GetRootQueueMetrics(), policy, mAgent, scheduler
.GetClusterResource(), 1L, res, scheduler.GetMinimumResourceCapability(), maxAlloc
, "dedicated", null, isMove);
// add a few reservations to the plan
long ts = Runtime.CurrentTimeMillis();
ReservationId r1 = ReservationId.NewInstance(ts, 1);
int[] f1 = new int[] { 10, 10, 10, 10, 10 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r1, null, "u3", "dedicated", 0, 0 + f1.Length, ReservationSystemTestUtil.GenerateAllocation
(0L, 1L, f1), res, minAlloc)));
ReservationId r2 = ReservationId.NewInstance(ts, 2);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r2, null, "u3", "dedicated", 3, 3 + f1.Length, ReservationSystemTestUtil.GenerateAllocation
(3L, 1L, f1), res, minAlloc)));
ReservationId r3 = ReservationId.NewInstance(ts, 3);
int[] f2 = new int[] { 0, 10, 20, 10, 0 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r3, null, "u4", "dedicated", 10, 10 + f2.Length, ReservationSystemTestUtil.GenerateAllocation
(10L, 1L, f2), res, minAlloc)));
AbstractSchedulerPlanFollower planFollower = CreatePlanFollower();
Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(0L);
planFollower.Run();
Queue q = GetReservationQueue(r1.ToString());
AssertReservationQueueExists(r1);
// submit an app to r1
string user_0 = "test-user";
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId appAttemptId_0 = ApplicationAttemptId.NewInstance(appId, 0);
AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, q.GetQueueName
(), user_0);
scheduler.Handle(addAppEvent);
AppAttemptAddedSchedulerEvent appAttemptAddedEvent = new AppAttemptAddedSchedulerEvent
(appAttemptId_0, false);
scheduler.Handle(appAttemptAddedEvent);
// initial default reservation queue should have no apps
Queue defQ = GetDefaultQueue();
NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ));
AssertReservationQueueExists(r1, 0.1, 0.1);
NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(q));
AssertReservationQueueDoesNotExist(r2);
AssertReservationQueueDoesNotExist(r3);
Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(3L);
planFollower.Run();
NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ));
AssertReservationQueueExists(r1, 0.1, 0.1);
NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(q));
AssertReservationQueueExists(r2, 0.1, 0.1);
AssertReservationQueueDoesNotExist(r3);
Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(10L);
planFollower.Run();
q = GetReservationQueue(r1.ToString());
if (isMove)
{
// app should have been moved to default reservation queue
NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(defQ));
NUnit.Framework.Assert.IsNull(q);
}
else
{
// app should be killed
NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ));
NUnit.Framework.Assert.IsNotNull(q);
AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = new AppAttemptRemovedSchedulerEvent
(appAttemptId_0, RMAppAttemptState.Killed, false);
scheduler.Handle(appAttemptRemovedEvent);
}
AssertReservationQueueDoesNotExist(r2);
AssertReservationQueueExists(r3, 0, 1.0);
Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(11L);
planFollower.Run();
if (isMove)
{
// app should have been moved to default reservation queue
NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(defQ));
}
else
{
// app should be killed
NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ));
}
AssertReservationQueueDoesNotExist(r1);
AssertReservationQueueDoesNotExist(r2);
AssertReservationQueueExists(r3, 0.1, 0.1);
Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(12L);
planFollower.Run();
AssertReservationQueueDoesNotExist(r1);
AssertReservationQueueDoesNotExist(r2);
AssertReservationQueueExists(r3, 0.2, 0.2);
Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(16L);
planFollower.Run();
AssertReservationQueueDoesNotExist(r1);
AssertReservationQueueDoesNotExist(r2);
AssertReservationQueueDoesNotExist(r3);
VerifyCapacity(defQ);
}
public virtual void TestReplanningPlanCapacityLoss()
{
Resource clusterCapacity = Resource.NewInstance(100 * 1024, 10);
Resource minAlloc = Resource.NewInstance(1024, 1);
Resource maxAlloc = Resource.NewInstance(1024 * 8, 8);
ResourceCalculator res = new DefaultResourceCalculator();
long step = 1L;
Clock clock = Org.Mockito.Mockito.Mock <Clock>();
ReservationAgent agent = Org.Mockito.Mockito.Mock <ReservationAgent>();
SharingPolicy policy = new NoOverCommitPolicy();
policy.Init("root.dedicated", null);
QueueMetrics queueMetrics = Org.Mockito.Mockito.Mock <QueueMetrics>();
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L);
SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
ReservationSchedulerConfiguration conf = Org.Mockito.Mockito.Mock <ReservationSchedulerConfiguration
>();
Org.Mockito.Mockito.When(conf.GetEnforcementWindow(Matchers.Any <string>())).ThenReturn
(6L);
enf.Init("blah", conf);
// Initialize the plan with more resources
InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity
, step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock);
// add reservation filling the plan (separating them 1ms, so we are sure
// s2 follows s1 on acceptance
long ts = Runtime.CurrentTimeMillis();
ReservationId r1 = ReservationId.NewInstance(ts, 1);
int[] f5 = new int[] { 20, 20, 20, 20, 20 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r1, null, "u3", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(1L);
ReservationId r2 = ReservationId.NewInstance(ts, 2);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r2, null, "u4", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(2L);
ReservationId r3 = ReservationId.NewInstance(ts, 3);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r3, null, "u5", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(3L);
ReservationId r4 = ReservationId.NewInstance(ts, 4);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r4, null, "u6", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(4L);
ReservationId r5 = ReservationId.NewInstance(ts, 5);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r5, null, "u7", "dedicated", 0, 0 + f5.Length, GenerateAllocation(0, f5), res,
minAlloc)));
int[] f6 = new int[] { 50, 50, 50, 50, 50 };
ReservationId r6 = ReservationId.NewInstance(ts, 6);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r6, null, "u3", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res
, minAlloc)));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(6L);
ReservationId r7 = ReservationId.NewInstance(ts, 7);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r7, null, "u4", "dedicated", 10, 10 + f6.Length, GenerateAllocation(10, f6), res
, minAlloc)));
// remove some of the resources (requires replanning)
plan.SetTotalCapacity(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance(70
* 1024, 70));
Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(0L);
// run the replanner
enf.Plan(plan, null);
// check which reservation are still present
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r1));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r2));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r3));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r6));
NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(r7));
// and which ones are removed
NUnit.Framework.Assert.IsNull(plan.GetReservationById(r4));
NUnit.Framework.Assert.IsNull(plan.GetReservationById(r5));
// check resources at each moment in time no more exceed capacity
for (int i = 0; i < 20; i++)
{
int tot = 0;
foreach (ReservationAllocation r in plan.GetReservationsAtTime(i))
{
tot = r.GetResourcesAtTime(i).GetMemory();
}
NUnit.Framework.Assert.IsTrue(tot <= 70 * 1024);
}
}
