/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Avro.AvroRemoteException"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestJobHistoryData()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
SleepJob sleepJob = new SleepJob();
sleepJob.SetConf(mrCluster.GetConfig());
// Job with 3 maps and 2 reduces
Job job = sleepJob.CreateJob(3, 2, 1000, 1, 500, 1);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.WaitForCompletion(true);
Counters counterMR = job.GetCounters();
JobId jobId = TypeConverter.ToYarn(job.GetJobID());
ApplicationId appID = jobId.GetAppId();
int pollElapsed = 0;
while (true)
{
Sharpen.Thread.Sleep(1000);
pollElapsed += 1000;
if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps
()[appID].GetState()))
{
break;
}
if (pollElapsed >= 60000)
{
Log.Warn("application did not reach terminal state within 60 seconds");
break;
}
}
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager
().GetRMContext().GetRMApps()[appID].GetState());
Counters counterHS = job.GetCounters();
//TODO the Assert below worked. need to check
//Should we compare each field or convert to V2 counter and compare
Log.Info("CounterHS " + counterHS);
Log.Info("CounterMR " + counterMR);
NUnit.Framework.Assert.AreEqual(counterHS, counterMR);
HSClientProtocol historyClient = InstantiateHistoryProxy();
GetJobReportRequest gjReq = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetJobReportRequest
>();
gjReq.SetJobId(jobId);
JobReport jobReport = historyClient.GetJobReport(gjReq).GetJobReport();
VerifyJobReport(jobReport, jobId);
}
/// <exception cref="System.IO.IOException"/>
public virtual GetJobReportResponse GetJobReport(GetJobReportRequest request)
{
MRServiceProtos.GetJobReportRequestProto requestProto = ((GetJobReportRequestPBImpl
)request).GetProto();
try
{
return(new GetJobReportResponsePBImpl(proxy.GetJobReport(null, requestProto)));
}
catch (ServiceException e)
{
throw UnwrapAndThrowException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual LogParams GetLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID
)
{
JobId jobId = TypeConverter.ToYarn(oldJobID);
GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest
>();
request.SetJobId(jobId);
JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest
), request)).GetJobReport();
if (EnumSet.Of(JobState.Succeeded, JobState.Failed, JobState.Killed, JobState.Error
).Contains(report.GetJobState()))
{
if (oldTaskAttemptID != null)
{
GetTaskAttemptReportRequest taRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
>();
taRequest.SetTaskAttemptId(TypeConverter.ToYarn(oldTaskAttemptID));
TaskAttemptReport taReport = ((GetTaskAttemptReportResponse)Invoke("getTaskAttemptReport"
, typeof(GetTaskAttemptReportRequest), taRequest)).GetTaskAttemptReport();
if (taReport.GetContainerId() == null || taReport.GetNodeManagerHost() == null)
{
throw new IOException("Unable to get log information for task: " + oldTaskAttemptID
);
}
return(new LogParams(taReport.GetContainerId().ToString(), taReport.GetContainerId
().GetApplicationAttemptId().GetApplicationId().ToString(), NodeId.NewInstance(taReport
.GetNodeManagerHost(), taReport.GetNodeManagerPort()).ToString(), report.GetUser
()));
}
else
{
if (report.GetAMInfos() == null || report.GetAMInfos().Count == 0)
{
throw new IOException("Unable to get log information for job: " + oldJobID);
}
AMInfo amInfo = report.GetAMInfos()[report.GetAMInfos().Count - 1];
return(new LogParams(amInfo.GetContainerId().ToString(), amInfo.GetAppAttemptId()
.GetApplicationId().ToString(), NodeId.NewInstance(amInfo.GetNodeManagerHost(),
amInfo.GetNodeManagerPort()).ToString(), report.GetUser()));
}
}
else
{
throw new IOException("Cannot get log path for a in-progress job");
}
}
/// <exception cref="System.IO.IOException"/>
public virtual GetJobReportResponse GetJobReport(GetJobReportRequest request)
{
JobReport jobReport = recordFactory.NewRecordInstance <JobReport>();
jobReport.SetJobId(request.GetJobId());
jobReport.SetJobState(jobState);
jobReport.SetUser(applicationReport.GetUser());
jobReport.SetStartTime(applicationReport.GetStartTime());
jobReport.SetDiagnostics(applicationReport.GetDiagnostics());
jobReport.SetJobName(applicationReport.GetName());
jobReport.SetTrackingUrl(applicationReport.GetTrackingUrl());
jobReport.SetFinishTime(applicationReport.GetFinishTime());
GetJobReportResponse resp = recordFactory.NewRecordInstance <GetJobReportResponse>
();
resp.SetJobReport(jobReport);
return(resp);
}
/// <exception cref="System.IO.IOException"/>
public virtual GetJobReportResponse GetJobReport(GetJobReportRequest request)
{
this._enclosing.amContact = true;
JobReport jobReport = TestClientRedirect.recordFactory.NewRecordInstance <JobReport
>();
jobReport.SetJobId(request.GetJobId());
jobReport.SetJobState(JobState.Running);
jobReport.SetJobName("TestClientRedirect-jobname");
jobReport.SetUser("TestClientRedirect-user");
jobReport.SetStartTime(0L);
jobReport.SetFinishTime(1L);
GetJobReportResponse response = TestClientRedirect.recordFactory.NewRecordInstance
<GetJobReportResponse>();
response.SetJobReport(jobReport);
return(response);
}
/// <exception cref="System.IO.IOException"/>
public virtual GetJobReportResponse GetJobReport(GetJobReportRequest request)
{
JobId jobId = request.GetJobId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = this.VerifyAndGetJob(jobId, false
);
GetJobReportResponse response = this.recordFactory.NewRecordInstance <GetJobReportResponse
>();
if (job != null)
{
response.SetJobReport(job.GetReport());
}
else
{
response.SetJobReport(null);
}
return(response);
}
/// <exception cref="System.IO.IOException"/>
public virtual JobStatus GetJobStatus(JobID oldJobID)
{
JobId jobId = TypeConverter.ToYarn(oldJobID);
GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest
>();
request.SetJobId(jobId);
JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest
), request)).GetJobReport();
JobStatus jobStatus = null;
if (report != null)
{
if (StringUtils.IsEmpty(report.GetJobFile()))
{
string jobFile = MRApps.GetJobFile(conf, report.GetUser(), oldJobID);
report.SetJobFile(jobFile);
}
string historyTrackingUrl = report.GetTrackingUrl();
string url = StringUtils.IsNotEmpty(historyTrackingUrl) ? historyTrackingUrl : trackingUrl;
jobStatus = TypeConverter.FromYarn(report, url);
}
return(jobStatus);
}
public virtual void Test()
{
TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService
(this, 1, 0, false);
Configuration conf = new Configuration();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(attempt, TaskAttemptState.Running);
// send the diagnostic
string diagnostic1 = "Diagnostic1";
string diagnostic2 = "Diagnostic2";
app.GetContext().GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attempt
.GetID(), diagnostic1));
// send the status update
TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
();
taskAttemptStatus.id = attempt.GetID();
taskAttemptStatus.progress = 0.5f;
taskAttemptStatus.stateString = "RUNNING";
taskAttemptStatus.taskState = TaskAttemptState.Running;
taskAttemptStatus.phase = Phase.Map;
// send the status update
app.GetContext().GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(attempt
.GetID(), taskAttemptStatus));
//verify that all object are fully populated by invoking RPCs.
YarnRPC rpc = YarnRPC.Create(conf);
MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol),
app.clientService.GetBindAddress(), conf);
GetCountersRequest gcRequest = recordFactory.NewRecordInstance <GetCountersRequest
>();
gcRequest.SetJobId(job.GetID());
NUnit.Framework.Assert.IsNotNull("Counters is null", proxy.GetCounters(gcRequest)
.GetCounters());
GetJobReportRequest gjrRequest = recordFactory.NewRecordInstance <GetJobReportRequest
>();
gjrRequest.SetJobId(job.GetID());
JobReport jr = proxy.GetJobReport(gjrRequest).GetJobReport();
VerifyJobReport(jr);
GetTaskAttemptCompletionEventsRequest gtaceRequest = recordFactory.NewRecordInstance
<GetTaskAttemptCompletionEventsRequest>();
gtaceRequest.SetJobId(job.GetID());
gtaceRequest.SetFromEventId(0);
gtaceRequest.SetMaxEvents(10);
NUnit.Framework.Assert.IsNotNull("TaskCompletionEvents is null", proxy.GetTaskAttemptCompletionEvents
(gtaceRequest).GetCompletionEventList());
GetDiagnosticsRequest gdRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest
>();
gdRequest.SetTaskAttemptId(attempt.GetID());
NUnit.Framework.Assert.IsNotNull("Diagnostics is null", proxy.GetDiagnostics(gdRequest
).GetDiagnosticsList());
GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
>();
gtarRequest.SetTaskAttemptId(attempt.GetID());
TaskAttemptReport tar = proxy.GetTaskAttemptReport(gtarRequest).GetTaskAttemptReport
();
VerifyTaskAttemptReport(tar);
GetTaskReportRequest gtrRequest = recordFactory.NewRecordInstance <GetTaskReportRequest
>();
gtrRequest.SetTaskId(task.GetID());
NUnit.Framework.Assert.IsNotNull("TaskReport is null", proxy.GetTaskReport(gtrRequest
).GetTaskReport());
GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest
>();
gtreportsRequest.SetJobId(job.GetID());
gtreportsRequest.SetTaskType(TaskType.Map);
NUnit.Framework.Assert.IsNotNull("TaskReports for map is null", proxy.GetTaskReports
(gtreportsRequest).GetTaskReportList());
gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>();
gtreportsRequest.SetJobId(job.GetID());
gtreportsRequest.SetTaskType(TaskType.Reduce);
NUnit.Framework.Assert.IsNotNull("TaskReports for reduce is null", proxy.GetTaskReports
(gtreportsRequest).GetTaskReportList());
IList <string> diag = proxy.GetDiagnostics(gdRequest).GetDiagnosticsList();
NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, diag.Count);
NUnit.Framework.Assert.AreEqual("Diag 1 not correct", diagnostic1, diag[0].ToString
());
TaskReport taskReport = proxy.GetTaskReport(gtrRequest).GetTaskReport();
NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, taskReport.GetDiagnosticsCount
());
//send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts()
.Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone));
app.WaitForState(job, JobState.Succeeded);
// For invalid jobid, throw IOException
gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>();
gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001"
)));
gtreportsRequest.SetTaskType(TaskType.Reduce);
try
{
proxy.GetTaskReports(gtreportsRequest);
NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id");
}
catch (IOException)
{
}
}
/// <exception cref="System.IO.IOException"/>
public virtual GetJobReportResponse GetJobReport(GetJobReportRequest request)
{
return(null);
}
public virtual void TestDelegationToken()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
YarnConfiguration conf = new YarnConfiguration(new JobConf());
// Just a random principle
conf.Set(JHAdminConfig.MrHistoryPrincipal, "RandomOrc/[email protected]");
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
long initialInterval = 10000l;
long maxLifetime = 20000l;
long renewInterval = 10000l;
JobHistoryServer jobHistoryServer = null;
MRClientProtocol clientUsingDT = null;
long tokenFetchTime;
try
{
jobHistoryServer = new _JobHistoryServer_87(initialInterval, maxLifetime, renewInterval
);
// no keytab based login
// Don't need it, skip.;
// final JobHistoryServer jobHistoryServer = jhServer;
jobHistoryServer.Init(conf);
jobHistoryServer.Start();
MRClientProtocol hsService = jobHistoryServer.GetClientService().GetClientHandler
();
// Fake the authentication-method
UserGroupInformation loggedInUser = UserGroupInformation.CreateRemoteUser("*****@*****.**"
);
NUnit.Framework.Assert.AreEqual("testrenewer", loggedInUser.GetShortUserName());
// Default realm is APACHE.ORG
loggedInUser.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
);
Token token = GetDelegationToken(loggedInUser, hsService, loggedInUser.GetShortUserName
());
tokenFetchTime = Runtime.CurrentTimeMillis();
Log.Info("Got delegation token at: " + tokenFetchTime);
// Now try talking to JHS using the delegation token
clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress
(), "TheDarkLord", conf);
GetJobReportRequest jobReportRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetJobReportRequest>();
jobReportRequest.SetJobId(MRBuilderUtils.NewJobId(123456, 1, 1));
try
{
clientUsingDT.GetJobReport(jobReportRequest);
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual("Unknown job job_123456_0001", e.Message);
}
// Renew after 50% of token age.
while (Runtime.CurrentTimeMillis() < tokenFetchTime + initialInterval / 2)
{
Sharpen.Thread.Sleep(500l);
}
long nextExpTime = RenewDelegationToken(loggedInUser, hsService, token);
long renewalTime = Runtime.CurrentTimeMillis();
Log.Info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime);
// Wait for first expiry, but before renewed expiry.
while (Runtime.CurrentTimeMillis() > tokenFetchTime + initialInterval && Runtime.
CurrentTimeMillis() < nextExpTime)
{
Sharpen.Thread.Sleep(500l);
}
Sharpen.Thread.Sleep(50l);
// Valid token because of renewal.
try
{
clientUsingDT.GetJobReport(jobReportRequest);
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual("Unknown job job_123456_0001", e.Message);
}
// Wait for expiry.
while (Runtime.CurrentTimeMillis() < renewalTime + renewInterval)
{
Sharpen.Thread.Sleep(500l);
}
Sharpen.Thread.Sleep(50l);
Log.Info("At time: " + Runtime.CurrentTimeMillis() + ", token should be invalid");
// Token should have expired.
try
{
clientUsingDT.GetJobReport(jobReportRequest);
NUnit.Framework.Assert.Fail("Should not have succeeded with an expired token");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException.Message.Contains("is expired"));
}
// Test cancellation
// Stop the existing proxy, start another.
if (clientUsingDT != null)
{
// RPC.stopProxy(clientUsingDT);
clientUsingDT = null;
}
token = GetDelegationToken(loggedInUser, hsService, loggedInUser.GetShortUserName
());
tokenFetchTime = Runtime.CurrentTimeMillis();
Log.Info("Got delegation token at: " + tokenFetchTime);
// Now try talking to HSService using the delegation token
clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress
(), "loginuser2", conf);
try
{
clientUsingDT.GetJobReport(jobReportRequest);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
CancelDelegationToken(loggedInUser, hsService, token);
// Testing the token with different renewer to cancel the token
Token tokenWithDifferentRenewer = GetDelegationToken(loggedInUser, hsService, "yarn"
);
CancelDelegationToken(loggedInUser, hsService, tokenWithDifferentRenewer);
if (clientUsingDT != null)
{
// RPC.stopProxy(clientUsingDT);
clientUsingDT = null;
}
// Creating a new connection.
clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress
(), "loginuser2", conf);
Log.Info("Cancelled delegation token at: " + Runtime.CurrentTimeMillis());
// Verify cancellation worked.
try
{
clientUsingDT.GetJobReport(jobReportRequest);
NUnit.Framework.Assert.Fail("Should not have succeeded with a cancelled delegation token"
);
}
catch (IOException)
{
}
}
finally
{
jobHistoryServer.Stop();
}
}
