public static ReservationDefinition GenerateBigRR(Random rand, long i)
{
rand.SetSeed(i);
long now = Runtime.CurrentTimeMillis();
// start time at random in the next 2 hours
long arrival = rand.Next(2 * 3600 * 1000);
// deadline at random in the next day
long deadline = rand.Next(24 * 3600 * 1000);
// create a request with a single atomic ask
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.SetArrival(now + arrival);
rr.SetDeadline(now + deadline);
int gang = 1;
int par = 100000;
// 100k tasks
long dur = rand.Next(60 * 1000);
// 1min tasks
ReservationRequest r = ReservationRequest.NewInstance(Resource.NewInstance(1024,
1), par, gang, dur);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.SetReservationResources(Collections.SingletonList(r));
rand.Next(3);
ReservationRequestInterpreter[] type = ReservationRequestInterpreter.Values();
reqs.SetInterpreter(type[rand.Next(type.Length)]);
rr.SetReservationRequests(reqs);
return(rr);
}
public static ReservationRequests NewInstance(IList <ReservationRequest> reservationResources
, ReservationRequestInterpreter type)
{
ReservationRequests reservationRequests = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ReservationRequests>();
reservationRequests.SetReservationResources(reservationResources);
reservationRequests.SetInterpreter(type);
return(reservationRequests);
}
public override void SetInterpreter(ReservationRequestInterpreter interpreter)
{
MaybeInitBuilder();
if (interpreter == null)
{
builder.ClearInterpreter();
return;
}
builder.SetInterpreter(ConvertToProtoFormat(interpreter));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.ContractValidationException
/// "/>
private bool ComputeAllocation(ReservationId reservationId, string user, Plan plan
, ReservationDefinition contract, ReservationAllocation oldReservation)
{
Log.Info("placing the following ReservationRequest: " + contract);
Resource totalCapacity = plan.GetTotalCapacity();
// Here we can addd logic to adjust the ResourceDefinition to account for
// system "imperfections" (e.g., scheduling delays for large containers).
// Align with plan step conservatively (i.e., ceil arrival, and floor
// deadline)
long earliestStart = contract.GetArrival();
long step = plan.GetStep();
if (earliestStart % step != 0)
{
earliestStart = earliestStart + (step - (earliestStart % step));
}
long deadline = contract.GetDeadline() - contract.GetDeadline() % plan.GetStep();
// setup temporary variables to handle time-relations between stages and
// intermediate answers
long curDeadline = deadline;
long oldDeadline = -1;
IDictionary <ReservationInterval, ReservationRequest> allocations = new Dictionary
<ReservationInterval, ReservationRequest>();
RLESparseResourceAllocation tempAssigned = new RLESparseResourceAllocation(plan.GetResourceCalculator
(), plan.GetMinimumAllocation());
IList <ReservationRequest> stages = contract.GetReservationRequests().GetReservationResources
();
ReservationRequestInterpreter type = contract.GetReservationRequests().GetInterpreter
();
// Iterate the stages in backward from deadline
for (ListIterator <ReservationRequest> li = stages.ListIterator(stages.Count); li.
HasPrevious();)
{
ReservationRequest currentReservationStage = li.Previous();
// validate the RR respect basic constraints
ValidateInput(plan, currentReservationStage, totalCapacity);
// run allocation for a single stage
IDictionary <ReservationInterval, ReservationRequest> curAlloc = PlaceSingleStage(
plan, tempAssigned, currentReservationStage, earliestStart, curDeadline, oldReservation
, totalCapacity);
if (curAlloc == null)
{
// if we did not find an allocation for the currentReservationStage
// return null, unless the ReservationDefinition we are placing is of
// type ANY
if (type != ReservationRequestInterpreter.RAny)
{
throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request"
);
}
else
{
continue;
}
}
else
{
// if we did find an allocation add it to the set of allocations
allocations.PutAll(curAlloc);
// if this request is of type ANY we are done searching (greedy)
// and can return the current allocation (break-out of the search)
if (type == ReservationRequestInterpreter.RAny)
{
break;
}
// if the request is of ORDER or ORDER_NO_GAP we constraint the next
// round of allocation to precede the current allocation, by setting
// curDeadline
if (type == ReservationRequestInterpreter.ROrder || type == ReservationRequestInterpreter
.ROrderNoGap)
{
curDeadline = FindEarliestTime(curAlloc.Keys);
// for ORDER_NO_GAP verify that the allocation found so far has no
// gap, return null otherwise (the greedy procedure failed to find a
// no-gap
// allocation)
if (type == ReservationRequestInterpreter.ROrderNoGap && oldDeadline > 0)
{
if (oldDeadline - FindLatestTime(curAlloc.Keys) > plan.GetStep())
{
throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request"
);
}
}
// keep the variable oldDeadline pointing to the last deadline we
// found
oldDeadline = curDeadline;
}
}
}
// / If we got here is because we failed to find an allocation for the
// ReservationDefinition give-up and report failure to the user
if (allocations.IsEmpty())
{
throw new PlanningException("The GreedyAgent" + " couldn't find a valid allocation for your request"
);
}
// create reservation with above allocations if not null/empty
ReservationRequest ZeroRes = ReservationRequest.NewInstance(Resource.NewInstance(
0, 0), 0);
long firstStartTime = FindEarliestTime(allocations.Keys);
// add zero-padding from arrival up to the first non-null allocation
// to guarantee that the reservation exists starting at arrival
if (firstStartTime > earliestStart)
{
allocations[new ReservationInterval(earliestStart, firstStartTime)] = ZeroRes;
firstStartTime = earliestStart;
}
// consider to add trailing zeros at the end for simmetry
// Actually add/update the reservation in the plan.
// This is subject to validation as other agents might be placing
// in parallel and there might be sharing policies the agent is not
// aware off.
ReservationAllocation capReservation = new InMemoryReservationAllocation(reservationId
, contract, user, plan.GetQueueName(), firstStartTime, FindLatestTime(allocations
.Keys), allocations, plan.GetResourceCalculator(), plan.GetMinimumAllocation());
if (oldReservation != null)
{
return(plan.UpdateReservation(capReservation));
}
else
{
return(plan.AddReservation(capReservation));
}
}
private YarnProtos.ReservationRequestInterpreterProto ConvertToProtoFormat(ReservationRequestInterpreter
r)
{
return(ProtoUtils.ConvertToProtoFormat(r));
}
public static ReservationRequestInterpreter ConvertFromProtoFormat(YarnProtos.ReservationRequestInterpreterProto
e)
{
return(ReservationRequestInterpreter.ValueOf(e.ToString()));
}
/*
* Reservation Request interpreter type
*/
public static YarnProtos.ReservationRequestInterpreterProto ConvertToProtoFormat(
ReservationRequestInterpreter e)
{
return(YarnProtos.ReservationRequestInterpreterProto.ValueOf(e.ToString()));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void ValidateReservationDefinition(ReservationId reservationId, ReservationDefinition
contract, Plan plan, string auditConstant)
{
string message = string.Empty;
// check if deadline is in the past
if (contract == null)
{
message = "Missing reservation definition." + " Please try again by specifying a reservation definition.";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
, "ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
if (contract.GetDeadline() <= clock.GetTime())
{
message = "The specified deadline: " + contract.GetDeadline() + " is the past. Please try again with deadline in the future.";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
, "ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
// Check if at least one RR has been specified
ReservationRequests resReqs = contract.GetReservationRequests();
if (resReqs == null)
{
message = "No resources have been specified to reserve." + "Please try again by specifying the resources to reserve.";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
, "ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
IList <ReservationRequest> resReq = resReqs.GetReservationResources();
if (resReq == null || resReq.IsEmpty())
{
message = "No resources have been specified to reserve." + " Please try again by specifying the resources to reserve.";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
, "ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
// compute minimum duration and max gang size
long minDuration = 0;
Resource maxGangSize = Resource.NewInstance(0, 0);
ReservationRequestInterpreter type = contract.GetReservationRequests().GetInterpreter
();
foreach (ReservationRequest rr in resReq)
{
if (type == ReservationRequestInterpreter.RAll || type == ReservationRequestInterpreter
.RAny)
{
minDuration = Math.Max(minDuration, rr.GetDuration());
}
else
{
minDuration += rr.GetDuration();
}
maxGangSize = Resources.Max(plan.GetResourceCalculator(), plan.GetTotalCapacity()
, maxGangSize, Resources.Multiply(rr.GetCapability(), rr.GetConcurrency()));
}
// verify the allocation is possible (skip for ANY)
if (contract.GetDeadline() - contract.GetArrival() < minDuration && type != ReservationRequestInterpreter
.RAny)
{
message = "The time difference (" + (contract.GetDeadline() - contract.GetArrival
()) + ") between arrival (" + contract.GetArrival() + ") " + "and deadline (" +
contract.GetDeadline() + ") must " + " be greater or equal to the minimum resource duration ("
+ minDuration + ")";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
, "ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
// check that the largest gang does not exceed the inventory available
// capacity (skip for ANY)
if (Resources.GreaterThan(plan.GetResourceCalculator(), plan.GetTotalCapacity(),
maxGangSize, plan.GetTotalCapacity()) && type != ReservationRequestInterpreter.RAny)
{
message = "The size of the largest gang in the reservation refinition (" + maxGangSize
+ ") exceed the capacity available (" + plan.GetTotalCapacity() + " )";
RMAuditLogger.LogFailure("UNKNOWN", auditConstant, "validate reservation input definition"
, "ClientRMService", message);
throw RPCUtil.GetRemoteException(message);
}
}
public abstract void SetInterpreter(ReservationRequestInterpreter interpreter);