/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public override void UnregisterApplicationMaster(FinalApplicationStatus appStatus
, string appMessage, string appTrackingUrl)
{
Preconditions.CheckArgument(appStatus != null, "AppStatus should not be null.");
FinishApplicationMasterRequest request = FinishApplicationMasterRequest.NewInstance
(appStatus, appMessage, appTrackingUrl);
try
{
while (true)
{
FinishApplicationMasterResponse response = rmClient.FinishApplicationMaster(request
);
if (response.GetIsUnregistered())
{
break;
}
Log.Info("Waiting for application to be successfully unregistered.");
Sharpen.Thread.Sleep(100);
}
}
catch (Exception)
{
Log.Info("Interrupted while waiting for application" + " to be removed from RMStateStore"
);
}
catch (ApplicationMasterNotRegisteredException)
{
Log.Warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing."
);
// re register with RM
RegisterApplicationMaster();
UnregisterApplicationMaster(appStatus, appMessage, appTrackingUrl);
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestFinishApplicationMasterOnHA()
{
FinishApplicationMasterRequest request = FinishApplicationMasterRequest.NewInstance
(FinalApplicationStatus.Succeeded, string.Empty, string.Empty);
FinishApplicationMasterResponse response = amClient.FinishApplicationMaster(request
);
NUnit.Framework.Assert.AreEqual(response, this.cluster.CreateFakeFinishApplicationMasterResponse
());
}
// Expected
// provide main method so this class can act as AM
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
if (args[0].Equals("success"))
{
ApplicationMasterProtocol client = ClientRMProxy.CreateRMProxy <ApplicationMasterProtocol
>(conf);
client.RegisterApplicationMaster(RegisterApplicationMasterRequest.NewInstance(NetUtils
.GetHostname(), -1, string.Empty));
Sharpen.Thread.Sleep(1000);
FinishApplicationMasterResponse resp = client.FinishApplicationMaster(FinishApplicationMasterRequest
.NewInstance(FinalApplicationStatus.Succeeded, "success", null));
NUnit.Framework.Assert.IsTrue(resp.GetIsUnregistered());
System.Environment.Exit(0);
}
else
{
System.Environment.Exit(1);
}
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual YarnServiceProtos.FinishApplicationMasterResponseProto FinishApplicationMaster
(RpcController arg0, YarnServiceProtos.FinishApplicationMasterRequestProto proto
)
{
FinishApplicationMasterRequestPBImpl request = new FinishApplicationMasterRequestPBImpl
(proto);
try
{
FinishApplicationMasterResponse response = real.FinishApplicationMaster(request);
return(((FinishApplicationMasterResponsePBImpl)response).GetProto());
}
catch (YarnException e)
{
throw new ServiceException(e);
}
catch (IOException e)
{
throw new ServiceException(e);
}
}
public virtual void TestTokenExpiry()
{
TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
();
TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf
, containerManager);
rm.Start();
Configuration conf = rm.GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
ApplicationMasterProtocol rmClient = null;
try
{
MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
RMApp app = rm.SubmitApp(1024);
nm1.NodeHeartbeat(true);
int waitCount = 0;
while (containerManager.containerTokens == null && waitCount++ < 20)
{
Log.Info("Waiting for AM Launch to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
// Create a client to the RM.
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
.ToString());
Credentials credentials = containerManager.GetContainerCredentials();
IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
.SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
currentUser.AddToken(amRMToken);
rmClient = CreateRMClient(rm, conf, rpc, currentUser);
RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
rmClient.RegisterApplicationMaster(request);
FinishApplicationMasterRequest finishAMRequest = Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <FinishApplicationMasterRequest>();
finishAMRequest.SetFinalApplicationStatus(FinalApplicationStatus.Succeeded);
finishAMRequest.SetDiagnostics("diagnostics");
finishAMRequest.SetTrackingUrl("url");
rmClient.FinishApplicationMaster(finishAMRequest);
// Send RMAppAttemptEventType.CONTAINER_FINISHED to transit RMAppAttempt
// from Finishing state to Finished State. Both AMRMToken and
// ClientToAMToken will be removed.
ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(attempt.GetMasterContainer
().GetId(), ContainerState.Complete, "AM Container Finished", 0);
rm.GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptContainerFinishedEvent
(applicationAttemptId, containerStatus, nm1.GetNodeId()));
// Make sure the RMAppAttempt is at Finished State.
// Both AMRMToken and ClientToAMToken have been removed.
int count = 0;
while (attempt.GetState() != RMAppAttemptState.Finished && count < maxWaitAttempts
)
{
Sharpen.Thread.Sleep(100);
count++;
}
NUnit.Framework.Assert.IsTrue(attempt.GetState() == RMAppAttemptState.Finished);
// Now simulate trying to allocate. RPC call itself should throw auth
// exception.
rpc.StopProxy(rmClient, conf);
// To avoid using cached client
rmClient = CreateRMClient(rm, conf, rpc, currentUser);
AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>();
try
{
rmClient.Allocate(allocateRequest);
NUnit.Framework.Assert.Fail("You got to be kidding me! " + "Using App tokens after app-finish should fail!"
);
}
catch (Exception t)
{
Log.Info("Exception found is ", t);
// The exception will still have the earlier appAttemptId as it picks it
// up from the token.
NUnit.Framework.Assert.IsTrue(t.InnerException.Message.Contains(applicationAttemptId
.ToString() + " not found in AMRMTokenSecretManager."));
}
}
finally
{
rm.Stop();
if (rmClient != null)
{
rpc.StopProxy(rmClient, conf);
}
}
}
protected internal virtual void DoUnregistration()
{
FinalApplicationStatus finishState = FinalApplicationStatus.Undefined;
JobImpl jobImpl = (JobImpl)job;
if (jobImpl.GetInternalState() == JobStateInternal.Succeeded)
{
finishState = FinalApplicationStatus.Succeeded;
}
else
{
if (jobImpl.GetInternalState() == JobStateInternal.Killed || (jobImpl.GetInternalState
() == JobStateInternal.Running && isSignalled))
{
finishState = FinalApplicationStatus.Killed;
}
else
{
if (jobImpl.GetInternalState() == JobStateInternal.Failed || jobImpl.GetInternalState
() == JobStateInternal.Error)
{
finishState = FinalApplicationStatus.Failed;
}
}
}
StringBuilder sb = new StringBuilder();
foreach (string s in job.GetDiagnostics())
{
sb.Append(s).Append("\n");
}
Log.Info("Setting job diagnostics to " + sb.ToString());
string historyUrl = MRWebAppUtil.GetApplicationWebURLOnJHSWithScheme(GetConfig(),
context.GetApplicationID());
Log.Info("History url is " + historyUrl);
FinishApplicationMasterRequest request = FinishApplicationMasterRequest.NewInstance
(finishState, sb.ToString(), historyUrl);
try
{
while (true)
{
FinishApplicationMasterResponse response = scheduler.FinishApplicationMaster(request
);
if (response.GetIsUnregistered())
{
// When excepting ClientService, other services are already stopped,
// it is safe to let clients know the final states. ClientService
// should wait for some time so clients have enough time to know the
// final states.
MRAppMaster.RunningAppContext raContext = (MRAppMaster.RunningAppContext)context;
raContext.MarkSuccessfulUnregistration();
break;
}
Log.Info("Waiting for application to be successfully unregistered.");
Sharpen.Thread.Sleep(rmPollInterval);
}
}
catch (ApplicationMasterNotRegisteredException)
{
// RM might have restarted or failed over and so lost the fact that AM had
// registered before.
Register();
DoUnregistration();
}
}
