public static ReservationDefinition GenerateBigRR(Random rand, long i) { rand.SetSeed(i); long now = Runtime.CurrentTimeMillis(); // start time at random in the next 2 hours long arrival = rand.Next(2 * 3600 * 1000); // deadline at random in the next day long deadline = rand.Next(24 * 3600 * 1000); // create a request with a single atomic ask ReservationDefinition rr = new ReservationDefinitionPBImpl(); rr.SetArrival(now + arrival); rr.SetDeadline(now + deadline); int gang = 1; int par = 100000; // 100k tasks long dur = rand.Next(60 * 1000); // 1min tasks ReservationRequest r = ReservationRequest.NewInstance(Resource.NewInstance(1024, 1), par, gang, dur); ReservationRequests reqs = new ReservationRequestsPBImpl(); reqs.SetReservationResources(Collections.SingletonList(r)); rand.Next(3); ReservationRequestInterpreter[] type = ReservationRequestInterpreter.Values(); reqs.SetInterpreter(type[rand.Next(type.Length)]); rr.SetReservationRequests(reqs); return(rr); }
public static ReservationRequests NewInstance(IList <ReservationRequest> reservationResources , ReservationRequestInterpreter type) { ReservationRequests reservationRequests = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ReservationRequests>(); reservationRequests.SetReservationResources(reservationResources); reservationRequests.SetInterpreter(type); return(reservationRequests); }
public override void SetInterpreter(ReservationRequestInterpreter interpreter) { MaybeInitBuilder(); if (interpreter == null) { builder.ClearInterpreter(); return; } builder.SetInterpreter(ConvertToProtoFormat(interpreter)); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException /// "/> /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.ContractValidationException /// "/> private bool ComputeAllocation(ReservationId reservationId, string user, Plan plan , ReservationDefinition contract, ReservationAllocation oldReservation) { Log.Info("placing the following ReservationRequest: " + contract); Resource totalCapacity = plan.GetTotalCapacity(); // Here we can addd logic to adjust the ResourceDefinition to account for // system "imperfections" (e.g., scheduling delays for large containers). // Align with plan step conservatively (i.e., ceil arrival, and floor // deadline) long earliestStart = contract.GetArrival(); long step = plan.GetStep(); if (earliestStart % step != 0) { earliestStart = earliestStart + (step - (earliestStart % step)); } long deadline = contract.GetDeadline() - contract.GetDeadline() % plan.GetStep(); // setup temporary variables to handle time-relations between stages and // intermediate answers long curDeadline = deadline; long oldDeadline = -1; IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary <ReservationInterval, ReservationRequest>(); RLESparseResourceAllocation tempAssigned = new RLESparseResourceAllocation(plan.GetResourceCalculator (), plan.GetMinimumAllocation()); IList <ReservationRequest> stages = contract.GetReservationRequests().GetReservationResources (); ReservationRequestInterpreter type = contract.GetReservationRequests().GetInterpreter (); // Iterate the stages in backward from deadline for (ListIterator <ReservationRequest> li = stages.ListIterator(stages.Count); li. HasPrevious();) { ReservationRequest currentReservationStage = li.Previous(); // validate the RR respect basic constraints ValidateInput(plan, currentReservationStage, totalCapacity); // run allocation for a single stage IDictionary <ReservationInterval, ReservationRequest> curAlloc = PlaceSingleStage( plan, tempAssigned, currentReservationStage, earliestStart, curDeadline, oldReservation , totalCapacity); if (curAlloc == null) { // if we did not find an allocation for the currentReservationStage // return null, unless the ReservationDefinition we are placing is of // type ANY if (type != ReservationRequestInterpreter.RAny) { throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request" ); } else { continue; } } else { // if we did find an allocation add it to the set of allocations allocations.PutAll(curAlloc); // if this request is of type ANY we are done searching (greedy) // and can return the current allocation (break-out of the search) if (type == ReservationRequestInterpreter.RAny) { break; } // if the request is of ORDER or ORDER_NO_GAP we constraint the next // round of allocation to precede the current allocation, by setting // curDeadline if (type == ReservationRequestInterpreter.ROrder || type == ReservationRequestInterpreter .ROrderNoGap) { curDeadline = FindEarliestTime(curAlloc.Keys); // for ORDER_NO_GAP verify that the allocation found so far has no // gap, return null otherwise (the greedy procedure failed to find a // no-gap // allocation) if (type == ReservationRequestInterpreter.ROrderNoGap && oldDeadline > 0) { if (oldDeadline - FindLatestTime(curAlloc.Keys) > plan.GetStep()) { throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request" ); } } // keep the variable oldDeadline pointing to the last deadline we // found oldDeadline = curDeadline; } } } // / If we got here is because we failed to find an allocation for the // ReservationDefinition give-up and report failure to the user if (allocations.IsEmpty()) { throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request" ); } // create reservation with above allocations if not null/empty ReservationRequest ZeroRes = ReservationRequest.NewInstance(Resource.NewInstance( 0, 0), 0); long firstStartTime = FindEarliestTime(allocations.Keys); // add zero-padding from arrival up to the first non-null allocation // to guarantee that the reservation exists starting at arrival if (firstStartTime > earliestStart) { allocations[new ReservationInterval(earliestStart, firstStartTime)] = ZeroRes; firstStartTime = earliestStart; } // consider to add trailing zeros at the end for simmetry // Actually add/update the reservation in the plan. // This is subject to validation as other agents might be placing // in parallel and there might be sharing policies the agent is not // aware off. ReservationAllocation capReservation = new InMemoryReservationAllocation(reservationId , contract, user, plan.GetQueueName(), firstStartTime, FindLatestTime(allocations .Keys), allocations, plan.GetResourceCalculator(), plan.GetMinimumAllocation()); if (oldReservation != null) { return(plan.UpdateReservation(capReservation)); } else { return(plan.AddReservation(capReservation)); } }
private YarnProtos.ReservationRequestInterpreterProto ConvertToProtoFormat(ReservationRequestInterpreter r) { return(ProtoUtils.ConvertToProtoFormat(r)); }
public static ReservationRequestInterpreter ConvertFromProtoFormat(YarnProtos.ReservationRequestInterpreterProto e) { return(ReservationRequestInterpreter.ValueOf(e.ToString())); }
/* * Reservation Request interpreter type */ public static YarnProtos.ReservationRequestInterpreterProto ConvertToProtoFormat( ReservationRequestInterpreter e) { return(YarnProtos.ReservationRequestInterpreterProto.ValueOf(e.ToString())); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private void ValidateReservationDefinition(ReservationId reservationId, ReservationDefinition contract, Plan plan, string auditConstant) { string message = string.Empty; // check if deadline is in the past if (contract == null) { message = "Missing reservation definition." + " Please try again by specifying a reservation definition."; RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition" , "ClientRMService", message); throw RPCUtil.GetRemoteException(message); } if (contract.GetDeadline() <= clock.GetTime()) { message = "The specified deadline: " + contract.GetDeadline() + " is the past. Please try again with deadline in the future."; RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition" , "ClientRMService", message); throw RPCUtil.GetRemoteException(message); } // Check if at least one RR has been specified ReservationRequests resReqs = contract.GetReservationRequests(); if (resReqs == null) { message = "No resources have been specified to reserve." + "Please try again by specifying the resources to reserve."; RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition" , "ClientRMService", message); throw RPCUtil.GetRemoteException(message); } IList <ReservationRequest> resReq = resReqs.GetReservationResources(); if (resReq == null || resReq.IsEmpty()) { message = "No resources have been specified to reserve." + " Please try again by specifying the resources to reserve."; RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition" , "ClientRMService", message); throw RPCUtil.GetRemoteException(message); } // compute minimum duration and max gang size long minDuration = 0; Resource maxGangSize = Resource.NewInstance(0, 0); ReservationRequestInterpreter type = contract.GetReservationRequests().GetInterpreter (); foreach (ReservationRequest rr in resReq) { if (type == ReservationRequestInterpreter.RAll || type == ReservationRequestInterpreter .RAny) { minDuration = Math.Max(minDuration, rr.GetDuration()); } else { minDuration += rr.GetDuration(); } maxGangSize = Resources.Max(plan.GetResourceCalculator(), plan.GetTotalCapacity() , maxGangSize, Resources.Multiply(rr.GetCapability(), rr.GetConcurrency())); } // verify the allocation is possible (skip for ANY) if (contract.GetDeadline() - contract.GetArrival() < minDuration && type != ReservationRequestInterpreter .RAny) { message = "The time difference (" + (contract.GetDeadline() - contract.GetArrival ()) + ") between arrival (" + contract.GetArrival() + ") " + "and deadline (" + contract.GetDeadline() + ") must " + " be greater or equal to the minimum resource duration (" + minDuration + ")"; RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition" , "ClientRMService", message); throw RPCUtil.GetRemoteException(message); } // check that the largest gang does not exceed the inventory available // capacity (skip for ANY) if (Resources.GreaterThan(plan.GetResourceCalculator(), plan.GetTotalCapacity(), maxGangSize, plan.GetTotalCapacity()) && type != ReservationRequestInterpreter.RAny) { message = "The size of the largest gang in the reservation refinition (" + maxGangSize + ") exceed the capacity available (" + plan.GetTotalCapacity() + " )"; RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition" , "ClientRMService", message); throw RPCUtil.GetRemoteException(message); } }
public abstract void SetInterpreter(ReservationRequestInterpreter interpreter);