public virtual void TestValidateResourceBlacklistRequest()
{
TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
();
TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new
YarnConfiguration(), containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = "*";
RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
WaitForLaunchedState(attempt);
// Create a client to the RM.
Configuration conf = rm.GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
.ToString());
Credentials credentials = containerManager.GetContainerCredentials();
IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
.SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
currentUser.AddToken(amRMToken);
ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc
, rmBindAddress, conf));
RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
client.RegisterApplicationMaster(request);
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
Sharpen.Collections.SingletonList(ResourceRequest.Any), null);
AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null
, blacklistRequest);
bool error = false;
try
{
client.Allocate(allocateRequest);
}
catch (InvalidResourceBlacklistRequestException)
{
error = true;
}
rm.Stop();
NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException"
, error);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
protected internal virtual AllocateResponse MakeRemoteRequest()
{
ApplyRequestLimits();
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
new AList <string>(blacklistAdditions), new AList <string>(blacklistRemovals));
AllocateRequest allocateRequest = AllocateRequest.NewInstance(lastResponseID, base
.GetApplicationProgress(), new AList <ResourceRequest>(ask), new AList <ContainerId
>(release), blacklistRequest);
AllocateResponse allocateResponse = scheduler.Allocate(allocateRequest);
lastResponseID = allocateResponse.GetResponseId();
availableResources = allocateResponse.GetAvailableResources();
lastClusterNmCount = clusterNmCount;
clusterNmCount = allocateResponse.GetNumClusterNodes();
int numCompletedContainers = allocateResponse.GetCompletedContainersStatuses().Count;
if (ask.Count > 0 || release.Count > 0)
{
Log.Info("getResources() for " + applicationId + ":" + " ask=" + ask.Count + " release= "
+ release.Count + " newContainers=" + allocateResponse.GetAllocatedContainers()
.Count + " finishedContainers=" + numCompletedContainers + " resourcelimit=" + availableResources
+ " knownNMs=" + clusterNmCount);
}
ask.Clear();
release.Clear();
if (numCompletedContainers > 0)
{
// re-send limited requests when a container completes to trigger asking
// for more containers
Sharpen.Collections.AddAll(requestLimitsToUpdate, requestLimits.Keys);
}
if (blacklistAdditions.Count > 0 || blacklistRemovals.Count > 0)
{
Log.Info("Update the blacklist for " + applicationId + ": blacklistAdditions=" +
blacklistAdditions.Count + " blacklistRemovals=" + blacklistRemovals.Count);
}
blacklistAdditions.Clear();
blacklistRemovals.Clear();
return(allocateResponse);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestAllocateOnHA()
{
AllocateRequest request = AllocateRequest.NewInstance(0, 50f, new AList <ResourceRequest
>(), new AList <ContainerId>(), ResourceBlacklistRequest.NewInstance(new AList <string
>(), new AList <string>()));
AllocateResponse response = amClient.Allocate(request);
NUnit.Framework.Assert.AreEqual(response, this.cluster.CreateFakeAllocateResponse
());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override AllocateResponse Allocate(float progressIndicator)
{
Preconditions.CheckArgument(progressIndicator >= 0, "Progress indicator should not be negative"
);
AllocateResponse allocateResponse = null;
IList <ResourceRequest> askList = null;
IList <ContainerId> releaseList = null;
AllocateRequest allocateRequest = null;
IList <string> blacklistToAdd = new AList <string>();
IList <string> blacklistToRemove = new AList <string>();
try
{
lock (this)
{
askList = new AList <ResourceRequest>(ask.Count);
foreach (ResourceRequest r in ask)
{
// create a copy of ResourceRequest as we might change it while the
// RPC layer is using it to send info across
askList.AddItem(ResourceRequest.NewInstance(r.GetPriority(), r.GetResourceName(),
r.GetCapability(), r.GetNumContainers(), r.GetRelaxLocality(), r.GetNodeLabelExpression
()));
}
releaseList = new AList <ContainerId>(release);
// optimistically clear this collection assuming no RPC failure
ask.Clear();
release.Clear();
Sharpen.Collections.AddAll(blacklistToAdd, blacklistAdditions);
Sharpen.Collections.AddAll(blacklistToRemove, blacklistRemovals);
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
blacklistToAdd, blacklistToRemove);
allocateRequest = AllocateRequest.NewInstance(lastResponseId, progressIndicator,
askList, releaseList, blacklistRequest);
// clear blacklistAdditions and blacklistRemovals before
// unsynchronized part
blacklistAdditions.Clear();
blacklistRemovals.Clear();
}
try
{
allocateResponse = rmClient.Allocate(allocateRequest);
}
catch (ApplicationMasterNotRegisteredException)
{
Log.Warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing."
);
lock (this)
{
Sharpen.Collections.AddAll(release, this.pendingRelease);
Sharpen.Collections.AddAll(blacklistAdditions, this.blacklistedNodes);
foreach (IDictionary <string, SortedDictionary <Resource, AMRMClientImpl.ResourceRequestInfo
> > rr in remoteRequestsTable.Values)
{
foreach (IDictionary <Resource, AMRMClientImpl.ResourceRequestInfo> capabalities in
rr.Values)
{
foreach (AMRMClientImpl.ResourceRequestInfo request in capabalities.Values)
{
AddResourceRequestToAsk(request.remoteRequest);
}
}
}
}
// re register with RM
RegisterApplicationMaster();
allocateResponse = Allocate(progressIndicator);
return(allocateResponse);
}
lock (this)
{
// update these on successful RPC
clusterNodeCount = allocateResponse.GetNumClusterNodes();
lastResponseId = allocateResponse.GetResponseId();
clusterAvailableResources = allocateResponse.GetAvailableResources();
if (!allocateResponse.GetNMTokens().IsEmpty())
{
PopulateNMTokens(allocateResponse.GetNMTokens());
}
if (allocateResponse.GetAMRMToken() != null)
{
UpdateAMRMToken(allocateResponse.GetAMRMToken());
}
if (!pendingRelease.IsEmpty() && !allocateResponse.GetCompletedContainersStatuses
().IsEmpty())
{
RemovePendingReleaseRequests(allocateResponse.GetCompletedContainersStatuses());
}
}
}
finally
{
// TODO how to differentiate remote yarn exception vs error in rpc
if (allocateResponse == null)
{
// we hit an exception in allocate()
// preserve ask and release for next call to allocate()
lock (this)
{
Sharpen.Collections.AddAll(release, releaseList);
// requests could have been added or deleted during call to allocate
// If requests were added/removed then there is nothing to do since
// the ResourceRequest object in ask would have the actual new value.
// If ask does not have this ResourceRequest then it was unchanged and
// so we can add the value back safely.
// This assumes that there will no concurrent calls to allocate() and
// so we dont have to worry about ask being changed in the
// synchronized block at the beginning of this method.
foreach (ResourceRequest oldAsk in askList)
{
if (!ask.Contains(oldAsk))
{
ask.AddItem(oldAsk);
}
}
Sharpen.Collections.AddAll(blacklistAdditions, blacklistToAdd);
Sharpen.Collections.AddAll(blacklistRemovals, blacklistToRemove);
}
}
}
return(allocateResponse);
}
