public static ReservationDefinition GenerateBigRR(Random rand, long i)
        {
            rand.SetSeed(i);
            long now = Runtime.CurrentTimeMillis();
            // start time at random in the next 2 hours
            long arrival = rand.Next(2 * 3600 * 1000);
            // deadline at random in the next day
            long deadline = rand.Next(24 * 3600 * 1000);
            // create a request with a single atomic ask
            ReservationDefinition rr = new ReservationDefinitionPBImpl();

            rr.SetArrival(now + arrival);
            rr.SetDeadline(now + deadline);
            int gang = 1;
            int par  = 100000;
            // 100k tasks
            long dur = rand.Next(60 * 1000);
            // 1min tasks
            ReservationRequest r = ReservationRequest.NewInstance(Resource.NewInstance(1024,
                                                                                       1), par, gang, dur);
            ReservationRequests reqs = new ReservationRequestsPBImpl();

            reqs.SetReservationResources(Collections.SingletonList(r));
            rand.Next(3);
            ReservationRequestInterpreter[] type = ReservationRequestInterpreter.Values();
            reqs.SetInterpreter(type[rand.Next(type.Length)]);
            rr.SetReservationRequests(reqs);
            return(rr);
        }
Beispiel #2
0
        public static ReservationRequests NewInstance(IList <ReservationRequest> reservationResources
                                                      , ReservationRequestInterpreter type)
        {
            ReservationRequests reservationRequests = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                      <ReservationRequests>();

            reservationRequests.SetReservationResources(reservationResources);
            reservationRequests.SetInterpreter(type);
            return(reservationRequests);
        }
Beispiel #3
0
 public override void SetInterpreter(ReservationRequestInterpreter interpreter)
 {
     MaybeInitBuilder();
     if (interpreter == null)
     {
         builder.ClearInterpreter();
         return;
     }
     builder.SetInterpreter(ConvertToProtoFormat(interpreter));
 }
Beispiel #4
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
        ///     "/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.ContractValidationException
        ///     "/>
        private bool ComputeAllocation(ReservationId reservationId, string user, Plan plan
                                       , ReservationDefinition contract, ReservationAllocation oldReservation)
        {
            Log.Info("placing the following ReservationRequest: " + contract);
            Resource totalCapacity = plan.GetTotalCapacity();
            // Here we can addd logic to adjust the ResourceDefinition to account for
            // system "imperfections" (e.g., scheduling delays for large containers).
            // Align with plan step conservatively (i.e., ceil arrival, and floor
            // deadline)
            long earliestStart = contract.GetArrival();
            long step          = plan.GetStep();

            if (earliestStart % step != 0)
            {
                earliestStart = earliestStart + (step - (earliestStart % step));
            }
            long deadline = contract.GetDeadline() - contract.GetDeadline() % plan.GetStep();
            // setup temporary variables to handle time-relations between stages and
            // intermediate answers
            long curDeadline = deadline;
            long oldDeadline = -1;
            IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary
                                                                                <ReservationInterval, ReservationRequest>();
            RLESparseResourceAllocation tempAssigned = new RLESparseResourceAllocation(plan.GetResourceCalculator
                                                                                           (), plan.GetMinimumAllocation());
            IList <ReservationRequest> stages = contract.GetReservationRequests().GetReservationResources
                                                    ();
            ReservationRequestInterpreter type = contract.GetReservationRequests().GetInterpreter
                                                     ();

            // Iterate the stages in backward from deadline
            for (ListIterator <ReservationRequest> li = stages.ListIterator(stages.Count); li.
                 HasPrevious();)
            {
                ReservationRequest currentReservationStage = li.Previous();
                // validate the RR respect basic constraints
                ValidateInput(plan, currentReservationStage, totalCapacity);
                // run allocation for a single stage
                IDictionary <ReservationInterval, ReservationRequest> curAlloc = PlaceSingleStage(
                    plan, tempAssigned, currentReservationStage, earliestStart, curDeadline, oldReservation
                    , totalCapacity);
                if (curAlloc == null)
                {
                    // if we did not find an allocation for the currentReservationStage
                    // return null, unless the ReservationDefinition we are placing is of
                    // type ANY
                    if (type != ReservationRequestInterpreter.RAny)
                    {
                        throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request"
                                                    );
                    }
                    else
                    {
                        continue;
                    }
                }
                else
                {
                    // if we did find an allocation add it to the set of allocations
                    allocations.PutAll(curAlloc);
                    // if this request is of type ANY we are done searching (greedy)
                    // and can return the current allocation (break-out of the search)
                    if (type == ReservationRequestInterpreter.RAny)
                    {
                        break;
                    }
                    // if the request is of ORDER or ORDER_NO_GAP we constraint the next
                    // round of allocation to precede the current allocation, by setting
                    // curDeadline
                    if (type == ReservationRequestInterpreter.ROrder || type == ReservationRequestInterpreter
                        .ROrderNoGap)
                    {
                        curDeadline = FindEarliestTime(curAlloc.Keys);
                        // for ORDER_NO_GAP verify that the allocation found so far has no
                        // gap, return null otherwise (the greedy procedure failed to find a
                        // no-gap
                        // allocation)
                        if (type == ReservationRequestInterpreter.ROrderNoGap && oldDeadline > 0)
                        {
                            if (oldDeadline - FindLatestTime(curAlloc.Keys) > plan.GetStep())
                            {
                                throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request"
                                                            );
                            }
                        }
                        // keep the variable oldDeadline pointing to the last deadline we
                        // found
                        oldDeadline = curDeadline;
                    }
                }
            }
            // / If we got here is because we failed to find an allocation for the
            // ReservationDefinition give-up and report failure to the user
            if (allocations.IsEmpty())
            {
                throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request"
                                            );
            }
            // create reservation with above allocations if not null/empty
            ReservationRequest ZeroRes = ReservationRequest.NewInstance(Resource.NewInstance(
                                                                            0, 0), 0);
            long firstStartTime = FindEarliestTime(allocations.Keys);

            // add zero-padding from arrival up to the first non-null allocation
            // to guarantee that the reservation exists starting at arrival
            if (firstStartTime > earliestStart)
            {
                allocations[new ReservationInterval(earliestStart, firstStartTime)] = ZeroRes;
                firstStartTime = earliestStart;
            }
            // consider to add trailing zeros at the end for simmetry
            // Actually add/update the reservation in the plan.
            // This is subject to validation as other agents might be placing
            // in parallel and there might be sharing policies the agent is not
            // aware off.
            ReservationAllocation capReservation = new InMemoryReservationAllocation(reservationId
                                                                                     , contract, user, plan.GetQueueName(), firstStartTime, FindLatestTime(allocations
                                                                                                                                                           .Keys), allocations, plan.GetResourceCalculator(), plan.GetMinimumAllocation());

            if (oldReservation != null)
            {
                return(plan.UpdateReservation(capReservation));
            }
            else
            {
                return(plan.AddReservation(capReservation));
            }
        }
Beispiel #5
0
 private YarnProtos.ReservationRequestInterpreterProto ConvertToProtoFormat(ReservationRequestInterpreter
                                                                            r)
 {
     return(ProtoUtils.ConvertToProtoFormat(r));
 }
Beispiel #6
0
 public static ReservationRequestInterpreter ConvertFromProtoFormat(YarnProtos.ReservationRequestInterpreterProto
                                                                    e)
 {
     return(ReservationRequestInterpreter.ValueOf(e.ToString()));
 }
Beispiel #7
0
 /*
  * Reservation Request interpreter type
  */
 public static YarnProtos.ReservationRequestInterpreterProto ConvertToProtoFormat(
     ReservationRequestInterpreter e)
 {
     return(YarnProtos.ReservationRequestInterpreterProto.ValueOf(e.ToString()));
 }
Beispiel #8
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        private void ValidateReservationDefinition(ReservationId reservationId, ReservationDefinition
                                                   contract, Plan plan, string auditConstant)
        {
            string message = string.Empty;

            // check if deadline is in the past
            if (contract == null)
            {
                message = "Missing reservation definition." + " Please try again by specifying a reservation definition.";
                RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
                                         , "ClientRMService", message);
                throw RPCUtil.GetRemoteException(message);
            }
            if (contract.GetDeadline() <= clock.GetTime())
            {
                message = "The specified deadline: " + contract.GetDeadline() + " is the past. Please try again with deadline in the future.";
                RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
                                         , "ClientRMService", message);
                throw RPCUtil.GetRemoteException(message);
            }
            // Check if at least one RR has been specified
            ReservationRequests resReqs = contract.GetReservationRequests();

            if (resReqs == null)
            {
                message = "No resources have been specified to reserve." + "Please try again by specifying the resources to reserve.";
                RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
                                         , "ClientRMService", message);
                throw RPCUtil.GetRemoteException(message);
            }
            IList <ReservationRequest> resReq = resReqs.GetReservationResources();

            if (resReq == null || resReq.IsEmpty())
            {
                message = "No resources have been specified to reserve." + " Please try again by specifying the resources to reserve.";
                RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
                                         , "ClientRMService", message);
                throw RPCUtil.GetRemoteException(message);
            }
            // compute minimum duration and max gang size
            long     minDuration = 0;
            Resource maxGangSize = Resource.NewInstance(0, 0);
            ReservationRequestInterpreter type = contract.GetReservationRequests().GetInterpreter
                                                     ();

            foreach (ReservationRequest rr in resReq)
            {
                if (type == ReservationRequestInterpreter.RAll || type == ReservationRequestInterpreter
                    .RAny)
                {
                    minDuration = Math.Max(minDuration, rr.GetDuration());
                }
                else
                {
                    minDuration += rr.GetDuration();
                }
                maxGangSize = Resources.Max(plan.GetResourceCalculator(), plan.GetTotalCapacity()
                                            , maxGangSize, Resources.Multiply(rr.GetCapability(), rr.GetConcurrency()));
            }
            // verify the allocation is possible (skip for ANY)
            if (contract.GetDeadline() - contract.GetArrival() < minDuration && type != ReservationRequestInterpreter
                .RAny)
            {
                message = "The time difference (" + (contract.GetDeadline() - contract.GetArrival
                                                         ()) + ") between arrival (" + contract.GetArrival() + ") " + "and deadline (" +
                          contract.GetDeadline() + ") must " + " be greater or equal to the minimum resource duration ("
                          + minDuration + ")";
                RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
                                         , "ClientRMService", message);
                throw RPCUtil.GetRemoteException(message);
            }
            // check that the largest gang does not exceed the inventory available
            // capacity (skip for ANY)
            if (Resources.GreaterThan(plan.GetResourceCalculator(), plan.GetTotalCapacity(),
                                      maxGangSize, plan.GetTotalCapacity()) && type != ReservationRequestInterpreter.RAny)
            {
                message = "The size of the largest gang in the reservation refinition (" + maxGangSize
                          + ") exceed the capacity available (" + plan.GetTotalCapacity() + " )";
                RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
                                         , "ClientRMService", message);
                throw RPCUtil.GetRemoteException(message);
            }
        }
Beispiel #9
0
 public abstract void SetInterpreter(ReservationRequestInterpreter interpreter);