public virtual void Move(Queue newQueue)
{
lock (this)
{
QueueMetrics oldMetrics = queue.GetMetrics();
QueueMetrics newMetrics = newQueue.GetMetrics();
foreach (IDictionary <string, ResourceRequest> asks in requests.Values)
{
ResourceRequest request = asks[ResourceRequest.Any];
if (request != null)
{
oldMetrics.DecrPendingResources(user, request.GetNumContainers(), request.GetCapability
());
newMetrics.IncrPendingResources(user, request.GetNumContainers(), request.GetCapability
());
}
}
oldMetrics.MoveAppFrom(this);
newMetrics.MoveAppTo(this);
activeUsersManager.DeactivateApplication(user, applicationId);
activeUsersManager = newQueue.GetActiveUsersManager();
activeUsersManager.ActivateApplication(user, applicationId);
this.queue = newQueue;
this.queueName = newQueue.GetQueueName();
}
}
public virtual void TestDefaultSingleQueueMetrics()
{
string queueName = "single";
string user = "******";
QueueMetrics metrics = QueueMetrics.ForQueue(ms, queueName, null, false, conf);
MetricsSource queueSource = QueueSource(ms, queueName);
AppSchedulingInfo app = MockApp(user);
metrics.SubmitApp(user);
MetricsSource userSource = UserSource(ms, queueName, user);
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100));
metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0
);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true);
CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
, 0);
metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2));
CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
, 0);
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
metrics.FinishApp(user, RMAppState.Finished);
CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true);
NUnit.Framework.Assert.IsNull(userSource);
}
/// <summary>
/// The ApplicationMaster is updating resource requirements for the
/// application, by asking for more resources and releasing resources acquired
/// by the application.
/// </summary>
/// <param name="requests">resources to be acquired</param>
/// <param name="recoverPreemptedRequest">recover Resource Request on preemption</param>
public virtual void UpdateResourceRequests(IList <ResourceRequest> requests, bool
recoverPreemptedRequest)
{
lock (this)
{
QueueMetrics metrics = queue.GetMetrics();
// Update resource requests
foreach (ResourceRequest request in requests)
{
Priority priority = request.GetPriority();
string resourceName = request.GetResourceName();
bool updatePendingResources = false;
ResourceRequest lastRequest = null;
if (resourceName.Equals(ResourceRequest.Any))
{
if (Log.IsDebugEnabled())
{
Log.Debug("update:" + " application=" + applicationId + " request=" + request);
}
updatePendingResources = true;
// Premature optimization?
// Assumes that we won't see more than one priority request updated
// in one call, reasonable assumption... however, it's totally safe
// to activate same application more than once.
// Thus we don't need another loop ala the one in decrementOutstanding()
// which is needed during deactivate.
if (request.GetNumContainers() > 0)
{
activeUsersManager.ActivateApplication(user, applicationId);
}
}
IDictionary <string, ResourceRequest> asks = this.requests[priority];
if (asks == null)
{
asks = new ConcurrentHashMap <string, ResourceRequest>();
this.requests[priority] = asks;
this.priorities.AddItem(priority);
}
lastRequest = asks[resourceName];
if (recoverPreemptedRequest && lastRequest != null)
{
// Increment the number of containers to 1, as it is recovering a
// single container.
request.SetNumContainers(lastRequest.GetNumContainers() + 1);
}
asks[resourceName] = request;
if (updatePendingResources)
{
// Similarly, deactivate application?
if (request.GetNumContainers() <= 0)
{
Log.Info("checking for deactivate of application :" + this.applicationId);
CheckForDeactivation();
}
int lastRequestContainers = lastRequest != null?lastRequest.GetNumContainers() :
0;
Org.Apache.Hadoop.Yarn.Api.Records.Resource lastRequestCapability = lastRequest !=
null?lastRequest.GetCapability() : Resources.None();
metrics.IncrPendingResources(user, request.GetNumContainers(), request.GetCapability
());
metrics.DecrPendingResources(user, lastRequestContainers, lastRequestCapability);
}
}
}
}
public virtual void TestTwoLevelWithUserMetrics()
{
string parentQueueName = "root";
string leafQueueName = "root.leaf";
string user = "******";
QueueMetrics parentMetrics = QueueMetrics.ForQueue(ms, parentQueueName, null, true
, conf);
Queue parentQueue = MockitoMaker.Make(MockitoMaker.Stub <Queue>().Returning(parentMetrics
).from.GetMetrics());
QueueMetrics metrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue, true
, conf);
MetricsSource parentQueueSource = QueueSource(ms, parentQueueName);
MetricsSource queueSource = QueueSource(ms, leafQueueName);
AppSchedulingInfo app = MockApp(user);
metrics.SubmitApp(user);
MetricsSource userSource = UserSource(ms, leafQueueName, user);
MetricsSource parentUserSource = UserSource(ms, parentQueueName, user);
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(userSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
metrics.SubmitAppAttempt(user);
CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true);
CheckApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true);
CheckApps(userSource, 1, 1, 0, 0, 0, 0, true);
CheckApps(parentUserSource, 1, 1, 0, 0, 0, 0, true);
parentMetrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100
));
metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100));
parentMetrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb,
10));
metrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10));
metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3));
CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0
);
CheckResources(parentQueueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0
, 0, 0);
CheckResources(userSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0, 0);
CheckResources(parentUserSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0
, 0);
metrics.RunAppAttempt(app.GetApplicationId(), user);
CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true);
CheckApps(userSource, 1, 0, 1, 0, 0, 0, true);
metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true);
metrics.ReserveResource(user, Resources.CreateResource(3 * Gb, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 3 *
Gb, 3, 1);
CheckResources(parentQueueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2
, 3 * Gb, 3, 1);
CheckResources(userSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb,
3, 1);
CheckResources(parentUserSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3
* Gb, 3, 1);
metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2));
metrics.UnreserveResource(user, Resources.CreateResource(3 * Gb, 3));
CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0
, 0);
CheckResources(parentQueueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2
, 0, 0, 0);
CheckResources(userSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0, 0, 0
);
CheckResources(parentUserSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0
, 0, 0);
metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser());
CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(userSource, 1, 0, 0, 0, 0, 0, true);
CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
metrics.FinishApp(user, RMAppState.Finished);
CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true);
CheckApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true);
CheckApps(userSource, 1, 0, 0, 1, 0, 0, true);
CheckApps(parentUserSource, 1, 0, 0, 1, 0, 0, true);
}
