public virtual void TestAddEmptyReservation() { Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc , minAlloc, maxAlloc, planName, replanner, true); ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId(); int[] alloc = new int[] { }; int start = 100; IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary <ReservationInterval, ReservationRequest>(); ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc .Length, alloc.Length, allocations.Values); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID , rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc ); NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID)); try { plan.AddReservation(rAllocation); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } }
public virtual void TestUpdateNonExistingReservation() { Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc , minAlloc, maxAlloc, planName, replanner, true); ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId(); // Try to update a reservation without adding int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 }; int start = 100; IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation (start, alloc, false); ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc .Length, alloc.Length, allocations.Values); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID , rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc ); NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID)); try { plan.UpdateReservation(rAllocation); NUnit.Framework.Assert.Fail("Update should fail as it does not exist in the plan" ); } catch (ArgumentException e) { NUnit.Framework.Assert.IsTrue(e.Message.EndsWith("does not exist in the plan")); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID)); }
public virtual void TestAddReservation() { Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc , minAlloc, maxAlloc, planName, replanner, true); ReservationId reservationID = ReservationSystemTestUtil.GetNewReservationId(); int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 }; int start = 100; IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation (start, alloc, false); ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc .Length, alloc.Length, allocations.Values); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID , rDef, user, planName, start, start + alloc.Length, allocations, resCalc, minAlloc ); NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID)); try { plan.AddReservation(rAllocation); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } DoAssertions(plan, rAllocation); for (int i = 0; i < alloc.Length; i++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc[i]), (alloc[i])), plan.GetTotalCommittedResources(start + i)); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc[i]), (alloc[i])), plan.GetConsumptionForUser(user, start + i)); } }
private static Tuple<DateTime, DateTime> GetReservationDateRange(DateRange dateRange, ReservationDefinition sd) { if (dateRange.DateFrom >= DateTime.Now) throw new ArgumentNullException("dateFrom"); if (dateRange.DateTo <= DateTime.Now) throw new ArgumentNullException("dateTo"); throw new NotImplementedException(); }
public static ReservationSubmissionRequest NewInstance(ReservationDefinition reservationDefinition , string queueName) { ReservationSubmissionRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ReservationSubmissionRequest>(); request.SetReservationDefinition(reservationDefinition); request.SetQueue(queueName); return(request); }
public override void SetReservationDefinition(ReservationDefinition reservationDefinition ) { MaybeInitBuilder(); if (reservationDefinition == null) { builder.ClearReservationDefinition(); } this.reservationDefinition = reservationDefinition; }
public static ReservationUpdateRequest NewInstance(ReservationDefinition reservationDefinition , ReservationId reservationId) { ReservationUpdateRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord < ReservationUpdateRequest>(); request.SetReservationDefinition(reservationDefinition); request.SetReservationId(reservationId); return(request); }
private static Tuple<DateTime, DateTime> GetReservationDateRange( DateRange dateRange, ReservationDefinition sd) { if (dateRange.DateFrom >= DateTime.Now) throw new ArgumentNullException("dateFrom"); if (dateRange.DateTo >= DateTime.Now) throw new ArgumentNullException("dateTo"); throw new NotImplementedException(); }
private void DoAssertions(ReservationAllocation rAllocation, ReservationId reservationID , ReservationDefinition rDef, IDictionary <ReservationInterval, ReservationRequest > allocations, int start, int[] alloc) { NUnit.Framework.Assert.AreEqual(reservationID, rAllocation.GetReservationId()); NUnit.Framework.Assert.AreEqual(rDef, rAllocation.GetReservationDefinition()); NUnit.Framework.Assert.AreEqual(allocations, rAllocation.GetAllocationRequests()); NUnit.Framework.Assert.AreEqual(user, rAllocation.GetUser()); NUnit.Framework.Assert.AreEqual(planName, rAllocation.GetPlanName()); NUnit.Framework.Assert.AreEqual(start, rAllocation.GetStartTime()); NUnit.Framework.Assert.AreEqual(start + alloc.Length + 1, rAllocation.GetEndTime( )); }
public override ReservationDefinition GetReservationDefinition() { YarnServiceProtos.ReservationUpdateRequestProtoOrBuilder p = viaProto ? proto : builder; if (reservationDefinition != null) { return(reservationDefinition); } if (!p.HasReservationDefinition()) { return(null); } reservationDefinition = ConvertFromProtoFormat(p.GetReservationDefinition()); return(reservationDefinition); }
public virtual void TestZeroAlloaction() { ReservationId reservationID = ReservationId.NewInstance(rand.NextLong(), rand.NextLong ()); int[] alloc = new int[] { }; long start = 0; ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc .Length + 1, alloc.Length); IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary <ReservationInterval, ReservationRequest>(); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID , rDef, user, planName, start, start + alloc.Length + 1, allocations, resCalc, minAlloc ); DoAssertions(rAllocation, reservationID, rDef, allocations, (int)start, alloc); NUnit.Framework.Assert.IsFalse(rAllocation.ContainsGangs()); }
internal InMemoryReservationAllocation(ReservationId reservationID, ReservationDefinition contract, string user, string planName, long startTime, long endTime, IDictionary <ReservationInterval, ReservationRequest> allocationRequests, ResourceCalculator calculator, Org.Apache.Hadoop.Yarn.Api.Records.Resource minAlloc) { this.contract = contract; this.startTime = startTime; this.endTime = endTime; this.reservationID = reservationID; this.user = user; this.allocationRequests = allocationRequests; this.planName = planName; resourcesOverTime = new RLESparseResourceAllocation(calculator, minAlloc); foreach (KeyValuePair <ReservationInterval, ReservationRequest> r in allocationRequests) { resourcesOverTime.AddInterval(r.Key, r.Value); if (r.Value.GetConcurrency() > 1) { hasGang = true; } } }
public virtual void TestBlocks() { ReservationId reservationID = ReservationId.NewInstance(rand.NextLong(), rand.NextLong ()); int[] alloc = new int[] { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = CreateSimpleReservationDefinition(start, start + alloc .Length + 1, alloc.Length); IDictionary <ReservationInterval, ReservationRequest> allocations = GenerateAllocation (start, alloc, false, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID , rDef, user, planName, start, start + alloc.Length + 1, allocations, resCalc, minAlloc ); DoAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); NUnit.Framework.Assert.IsFalse(rAllocation.ContainsGangs()); for (int i = 0; i < alloc.Length; i++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc[i]), (alloc[i])), rAllocation.GetResourcesAtTime(start + i)); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException /// "/> public virtual bool CreateReservation(ReservationId reservationId, string user, Plan plan, ReservationDefinition contract) { return(ComputeAllocation(reservationId, user, plan, contract, null)); }
public virtual void TestArchiveCompletedReservations() { Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc , minAlloc, maxAlloc, planName, replanner, true); ReservationId reservationID1 = ReservationSystemTestUtil.GetNewReservationId(); // First add a reservation int[] alloc1 = new int[] { 10, 10, 10, 10, 10, 10 }; int start = 100; IDictionary <ReservationInterval, ReservationRequest> allocations1 = GenerateAllocation (start, alloc1, false); ReservationDefinition rDef1 = CreateSimpleReservationDefinition(start, start + alloc1 .Length, alloc1.Length, allocations1.Values); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID1 , rDef1, user, planName, start, start + alloc1.Length, allocations1, resCalc, minAlloc ); NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID1)); try { plan.AddReservation(rAllocation); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } DoAssertions(plan, rAllocation); for (int i = 0; i < alloc1.Length; i++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc1[i]), (alloc1[i])), plan.GetTotalCommittedResources(start + i)); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc1[i]), (alloc1[i])), plan.GetConsumptionForUser(user, start + i)); } // Now add another one ReservationId reservationID2 = ReservationSystemTestUtil.GetNewReservationId(); int[] alloc2 = new int[] { 0, 5, 10, 5, 0 }; IDictionary <ReservationInterval, ReservationRequest> allocations2 = GenerateAllocation (start, alloc2, true); ReservationDefinition rDef2 = CreateSimpleReservationDefinition(start, start + alloc2 .Length, alloc2.Length, allocations2.Values); rAllocation = new InMemoryReservationAllocation(reservationID2, rDef2, user, planName , start, start + alloc2.Length, allocations2, resCalc, minAlloc); NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID2)); try { plan.AddReservation(rAllocation); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(reservationID2)); for (int i_1 = 0; i_1 < alloc2.Length; i_1++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc1[i_1] + alloc2[i_1] + i_1), alloc1[i_1] + alloc2[i_1] + i_1), plan .GetTotalCommittedResources(start + i_1)); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc1[i_1] + alloc2[i_1] + i_1), alloc1[i_1] + alloc2[i_1] + i_1), plan .GetConsumptionForUser(user, start + i_1)); } // Now archive completed reservations Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(106L); Org.Mockito.Mockito.When(policy.GetValidWindow()).ThenReturn(1L); try { // will only remove 2nd reservation as only that has fallen out of the // archival window plan.ArchiveCompletedReservations(clock.GetTime()); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } NUnit.Framework.Assert.IsNotNull(plan.GetReservationById(reservationID1)); NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID2)); for (int i_2 = 0; i_2 < alloc1.Length; i_2++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc1[i_2]), (alloc1[i_2])), plan.GetTotalCommittedResources(start + i_2 )); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (1024 * (alloc1[i_2]), (alloc1[i_2])), plan.GetConsumptionForUser(user, start + i_2)); } Org.Mockito.Mockito.When(clock.GetTime()).ThenReturn(107L); try { // will remove 1st reservation also as it has fallen out of the archival // window plan.ArchiveCompletedReservations(clock.GetTime()); } catch (PlanningException e) { NUnit.Framework.Assert.Fail(e.Message); } NUnit.Framework.Assert.IsNull(plan.GetReservationById(reservationID1)); for (int i_3 = 0; i_3 < alloc1.Length; i_3++) { NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (0, 0), plan.GetTotalCommittedResources(start + i_3)); NUnit.Framework.Assert.AreEqual(Org.Apache.Hadoop.Yarn.Api.Records.Resource.NewInstance (0, 0), plan.GetConsumptionForUser(user, start + i_3)); } }
private YarnProtos.ReservationDefinitionProto ConvertToProtoFormat(ReservationDefinition r) { return(((ReservationDefinitionPBImpl)r).GetProto()); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException /// "/> /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.ContractValidationException /// "/> private bool ComputeAllocation(ReservationId reservationId, string user, Plan plan , ReservationDefinition contract, ReservationAllocation oldReservation) { Log.Info("placing the following ReservationRequest: " + contract); Resource totalCapacity = plan.GetTotalCapacity(); // Here we can addd logic to adjust the ResourceDefinition to account for // system "imperfections" (e.g., scheduling delays for large containers). // Align with plan step conservatively (i.e., ceil arrival, and floor // deadline) long earliestStart = contract.GetArrival(); long step = plan.GetStep(); if (earliestStart % step != 0) { earliestStart = earliestStart + (step - (earliestStart % step)); } long deadline = contract.GetDeadline() - contract.GetDeadline() % plan.GetStep(); // setup temporary variables to handle time-relations between stages and // intermediate answers long curDeadline = deadline; long oldDeadline = -1; IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary <ReservationInterval, ReservationRequest>(); RLESparseResourceAllocation tempAssigned = new RLESparseResourceAllocation(plan.GetResourceCalculator (), plan.GetMinimumAllocation()); IList <ReservationRequest> stages = contract.GetReservationRequests().GetReservationResources (); ReservationRequestInterpreter type = contract.GetReservationRequests().GetInterpreter (); // Iterate the stages in backward from deadline for (ListIterator <ReservationRequest> li = stages.ListIterator(stages.Count); li. HasPrevious();) { ReservationRequest currentReservationStage = li.Previous(); // validate the RR respect basic constraints ValidateInput(plan, currentReservationStage, totalCapacity); // run allocation for a single stage IDictionary <ReservationInterval, ReservationRequest> curAlloc = PlaceSingleStage( plan, tempAssigned, currentReservationStage, earliestStart, curDeadline, oldReservation , totalCapacity); if (curAlloc == null) { // if we did not find an allocation for the currentReservationStage // return null, unless the ReservationDefinition we are placing is of // type ANY if (type != ReservationRequestInterpreter.RAny) { throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request" ); } else { continue; } } else { // if we did find an allocation add it to the set of allocations allocations.PutAll(curAlloc); // if this request is of type ANY we are done searching (greedy) // and can return the current allocation (break-out of the search) if (type == ReservationRequestInterpreter.RAny) { break; } // if the request is of ORDER or ORDER_NO_GAP we constraint the next // round of allocation to precede the current allocation, by setting // curDeadline if (type == ReservationRequestInterpreter.ROrder || type == ReservationRequestInterpreter .ROrderNoGap) { curDeadline = FindEarliestTime(curAlloc.Keys); // for ORDER_NO_GAP verify that the allocation found so far has no // gap, return null otherwise (the greedy procedure failed to find a // no-gap // allocation) if (type == ReservationRequestInterpreter.ROrderNoGap && oldDeadline > 0) { if (oldDeadline - FindLatestTime(curAlloc.Keys) > plan.GetStep()) { throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request" ); } } // keep the variable oldDeadline pointing to the last deadline we // found oldDeadline = curDeadline; } } } // / If we got here is because we failed to find an allocation for the // ReservationDefinition give-up and report failure to the user if (allocations.IsEmpty()) { throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request" ); } // create reservation with above allocations if not null/empty ReservationRequest ZeroRes = ReservationRequest.NewInstance(Resource.NewInstance( 0, 0), 0); long firstStartTime = FindEarliestTime(allocations.Keys); // add zero-padding from arrival up to the first non-null allocation // to guarantee that the reservation exists starting at arrival if (firstStartTime > earliestStart) { allocations[new ReservationInterval(earliestStart, firstStartTime)] = ZeroRes; firstStartTime = earliestStart; } // consider to add trailing zeros at the end for simmetry // Actually add/update the reservation in the plan. // This is subject to validation as other agents might be placing // in parallel and there might be sharing policies the agent is not // aware off. ReservationAllocation capReservation = new InMemoryReservationAllocation(reservationId , contract, user, plan.GetQueueName(), firstStartTime, FindLatestTime(allocations .Keys), allocations, plan.GetResourceCalculator(), plan.GetMinimumAllocation()); if (oldReservation != null) { return(plan.UpdateReservation(capReservation)); } else { return(plan.AddReservation(capReservation)); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException /// "/> public virtual bool UpdateReservation(ReservationId reservationId, string user, Plan plan, ReservationDefinition contract) { return(ComputeAllocation(reservationId, user, plan, contract, plan.GetReservationById (reservationId))); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private void ValidateReservationDefinition(ReservationId reservationId, ReservationDefinition contract, Plan plan, string auditConstant) { string message = string.Empty; // check if deadline is in the past if (contract == null) { message = "Missing reservation definition." + " Please try again by specifying a reservation definition."; RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition" , "ClientRMService", message); throw RPCUtil.GetRemoteException(message); } if (contract.GetDeadline() <= clock.GetTime()) { message = "The specified deadline: " + contract.GetDeadline() + " is the past. Please try again with deadline in the future."; RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition" , "ClientRMService", message); throw RPCUtil.GetRemoteException(message); } // Check if at least one RR has been specified ReservationRequests resReqs = contract.GetReservationRequests(); if (resReqs == null) { message = "No resources have been specified to reserve." + "Please try again by specifying the resources to reserve."; RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition" , "ClientRMService", message); throw RPCUtil.GetRemoteException(message); } IList <ReservationRequest> resReq = resReqs.GetReservationResources(); if (resReq == null || resReq.IsEmpty()) { message = "No resources have been specified to reserve." + " Please try again by specifying the resources to reserve."; RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition" , "ClientRMService", message); throw RPCUtil.GetRemoteException(message); } // compute minimum duration and max gang size long minDuration = 0; Resource maxGangSize = Resource.NewInstance(0, 0); ReservationRequestInterpreter type = contract.GetReservationRequests().GetInterpreter (); foreach (ReservationRequest rr in resReq) { if (type == ReservationRequestInterpreter.RAll || type == ReservationRequestInterpreter .RAny) { minDuration = Math.Max(minDuration, rr.GetDuration()); } else { minDuration += rr.GetDuration(); } maxGangSize = Resources.Max(plan.GetResourceCalculator(), plan.GetTotalCapacity() , maxGangSize, Resources.Multiply(rr.GetCapability(), rr.GetConcurrency())); } // verify the allocation is possible (skip for ANY) if (contract.GetDeadline() - contract.GetArrival() < minDuration && type != ReservationRequestInterpreter .RAny) { message = "The time difference (" + (contract.GetDeadline() - contract.GetArrival ()) + ") between arrival (" + contract.GetArrival() + ") " + "and deadline (" + contract.GetDeadline() + ") must " + " be greater or equal to the minimum resource duration (" + minDuration + ")"; RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition" , "ClientRMService", message); throw RPCUtil.GetRemoteException(message); } // check that the largest gang does not exceed the inventory available // capacity (skip for ANY) if (Resources.GreaterThan(plan.GetResourceCalculator(), plan.GetTotalCapacity(), maxGangSize, plan.GetTotalCapacity()) && type != ReservationRequestInterpreter.RAny) { message = "The size of the largest gang in the reservation refinition (" + maxGangSize + ") exceed the capacity available (" + plan.GetTotalCapacity() + " )"; RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition" , "ClientRMService", message); throw RPCUtil.GetRemoteException(message); } }
private static Tuple <DateTime, DateTime> GetReservationDateRange(ReservationRangeDTO reservation, ReservationDefinition sd) { if (reservation.DateFrom >= DateTime.Now) { throw new ArgumentNullException("dateFrom"); } if (reservation.DateTo <= DateTime.Now) { throw new ArgumentNullException("dateTo"); } throw new NotImplementedException(); }
public abstract void SetReservationDefinition(ReservationDefinition reservationDefinition );