public virtual void TestToJobID()
{
JobId jid = MRApps.ToJobID("job_1_1");
NUnit.Framework.Assert.AreEqual(1, jid.GetAppId().GetClusterTimestamp());
NUnit.Framework.Assert.AreEqual(1, jid.GetAppId().GetId());
NUnit.Framework.Assert.AreEqual(1, jid.GetId());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Avro.AvroRemoteException"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestJobHistoryData()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
SleepJob sleepJob = new SleepJob();
sleepJob.SetConf(mrCluster.GetConfig());
// Job with 3 maps and 2 reduces
Job job = sleepJob.CreateJob(3, 2, 1000, 1, 500, 1);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.WaitForCompletion(true);
Counters counterMR = job.GetCounters();
JobId jobId = TypeConverter.ToYarn(job.GetJobID());
ApplicationId appID = jobId.GetAppId();
int pollElapsed = 0;
while (true)
{
Sharpen.Thread.Sleep(1000);
pollElapsed += 1000;
if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps
()[appID].GetState()))
{
break;
}
if (pollElapsed >= 60000)
{
Log.Warn("application did not reach terminal state within 60 seconds");
break;
}
}
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager
().GetRMContext().GetRMApps()[appID].GetState());
Counters counterHS = job.GetCounters();
//TODO the Assert below worked. need to check
//Should we compare each field or convert to V2 counter and compare
Log.Info("CounterHS " + counterHS);
Log.Info("CounterMR " + counterMR);
NUnit.Framework.Assert.AreEqual(counterHS, counterMR);
HSClientProtocol historyClient = InstantiateHistoryProxy();
GetJobReportRequest gjReq = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetJobReportRequest
>();
gjReq.SetJobId(jobId);
JobReport jobReport = historyClient.GetJobReport(gjReq).GetJobReport();
VerifyJobReport(jobReport, jobId);
}
private void VerifyJobReport(JobReport jobReport, JobId jobId)
{
IList <AMInfo> amInfos = jobReport.GetAMInfos();
NUnit.Framework.Assert.AreEqual(1, amInfos.Count);
AMInfo amInfo = amInfos[0];
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(jobId.GetAppId
(), 1);
ContainerId amContainerId = ContainerId.NewContainerId(appAttemptId, 1);
NUnit.Framework.Assert.AreEqual(appAttemptId, amInfo.GetAppAttemptId());
NUnit.Framework.Assert.AreEqual(amContainerId, amInfo.GetContainerId());
NUnit.Framework.Assert.IsTrue(jobReport.GetSubmitTime() > 0);
NUnit.Framework.Assert.IsTrue(jobReport.GetStartTime() > 0 && jobReport.GetStartTime
() >= jobReport.GetSubmitTime());
NUnit.Framework.Assert.IsTrue(jobReport.GetFinishTime() > 0 && jobReport.GetFinishTime
() >= jobReport.GetStartTime());
}
public virtual void TestHistoryServerNotConfigured()
{
//RM doesn't have app report and job History Server is not configured
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(null, GetRMDelegate
());
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetUsername());
NUnit.Framework.Assert.AreEqual(JobStatus.State.Prep, jobStatus.GetState());
//RM has app report and job History Server is not configured
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
ApplicationReport applicationReport = GetFinishedApplicationReport();
Org.Mockito.Mockito.When(rm.GetApplicationReport(jobId.GetAppId())).ThenReturn(applicationReport
);
clientServiceDelegate = GetClientServiceDelegate(null, rm);
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.AreEqual(applicationReport.GetUser(), jobStatus.GetUsername
());
NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, jobStatus.GetState());
}
public static JobID FromYarn(JobId id)
{
string identifier = FromClusterTimeStamp(id.GetAppId().GetClusterTimestamp());
return(new JobID(identifier, id.GetId()));
}
/// <exception cref="System.Exception"/>
private void TestProfilerInternal(bool useDefault)
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
SleepJob sleepJob = new SleepJob();
JobConf sleepConf = new JobConf(mrCluster.GetConfig());
sleepConf.SetProfileEnabled(true);
sleepConf.SetProfileTaskRange(true, ProfiledTaskId.ToString());
sleepConf.SetProfileTaskRange(false, ProfiledTaskId.ToString());
if (!useDefault)
{
// use hprof for map to profile.out
sleepConf.Set(MRJobConfig.TaskMapProfileParams, "-agentlib:hprof=cpu=times,heap=sites,force=n,thread=y,verbose=n,"
+ "file=%s");
// use Xprof for reduce to stdout
sleepConf.Set(MRJobConfig.TaskReduceProfileParams, "-Xprof");
}
sleepJob.SetConf(sleepConf);
// 2-map-2-reduce SleepJob
Job job = sleepJob.CreateJob(2, 2, 500, 1, 500, 1);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.WaitForCompletion(true);
JobId jobId = TypeConverter.ToYarn(job.GetJobID());
ApplicationId appID = jobId.GetAppId();
int pollElapsed = 0;
while (true)
{
Sharpen.Thread.Sleep(1000);
pollElapsed += 1000;
if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps
()[appID].GetState()))
{
break;
}
if (pollElapsed >= 60000)
{
Log.Warn("application did not reach terminal state within 60 seconds");
break;
}
}
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager
().GetRMContext().GetRMApps()[appID].GetState());
// Job finished, verify logs
//
Configuration nmConf = mrCluster.GetNodeManager(0).GetConfig();
string appIdStr = appID.ToString();
string appIdSuffix = Sharpen.Runtime.Substring(appIdStr, "application_".Length, appIdStr
.Length);
string containerGlob = "container_" + appIdSuffix + "_*_*";
IDictionary <TaskAttemptID, Path> taLogDirs = new Dictionary <TaskAttemptID, Path>(
);
Sharpen.Pattern taskPattern = Sharpen.Pattern.Compile(".*Task:(attempt_" + appIdSuffix
+ "_[rm]_" + "[0-9]+_[0-9]+).*");
foreach (string logDir in nmConf.GetTrimmedStrings(YarnConfiguration.NmLogDirs))
{
// filter out MRAppMaster and create attemptId->logDir map
//
foreach (FileStatus fileStatus in localFs.GlobStatus(new Path(logDir + Path.Separator
+ appIdStr + Path.Separator + containerGlob + Path.Separator + TaskLog.LogName.
Syslog)))
{
BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(fileStatus
.GetPath())));
string line;
while ((line = br.ReadLine()) != null)
{
Matcher m = taskPattern.Matcher(line);
if (m.Matches())
{
// found Task done message
taLogDirs[TaskAttemptID.ForName(m.Group(1))] = fileStatus.GetPath().GetParent();
break;
}
}
br.Close();
}
}
NUnit.Framework.Assert.AreEqual(4, taLogDirs.Count);
// all 4 attempts found
foreach (KeyValuePair <TaskAttemptID, Path> dirEntry in taLogDirs)
{
TaskAttemptID tid = dirEntry.Key;
Path profilePath = new Path(dirEntry.Value, TaskLog.LogName.Profile.ToString());
Path stdoutPath = new Path(dirEntry.Value, TaskLog.LogName.Stdout.ToString());
if (useDefault || tid.GetTaskType() == TaskType.Map)
{
if (tid.GetTaskID().GetId() == ProfiledTaskId)
{
// verify profile.out
BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(profilePath
)));
string line = br.ReadLine();
NUnit.Framework.Assert.IsTrue("No hprof content found!", line != null && line.StartsWith
("JAVA PROFILE"));
br.Close();
NUnit.Framework.Assert.AreEqual(0L, localFs.GetFileStatus(stdoutPath).GetLen());
}
else
{
NUnit.Framework.Assert.IsFalse("hprof file should not exist", localFs.Exists(profilePath
));
}
}
else
{
NUnit.Framework.Assert.IsFalse("hprof file should not exist", localFs.Exists(profilePath
));
if (tid.GetTaskID().GetId() == ProfiledTaskId)
{
// reducer is profiled with Xprof
BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(stdoutPath
)));
bool flatProfFound = false;
string line;
while ((line = br.ReadLine()) != null)
{
if (line.StartsWith("Flat profile"))
{
flatProfFound = true;
break;
}
}
br.Close();
NUnit.Framework.Assert.IsTrue("Xprof flat profile not found!", flatProfFound);
}
else
{
NUnit.Framework.Assert.AreEqual(0L, localFs.GetFileStatus(stdoutPath).GetLen());
}
}
}
}
// TODO later: add explicit "isUber()" checks of some sort
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestContainerRollingLog()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
SleepJob sleepJob = new SleepJob();
JobConf sleepConf = new JobConf(mrCluster.GetConfig());
sleepConf.Set(MRJobConfig.MapLogLevel, Level.All.ToString());
long userLogKb = 4;
sleepConf.SetLong(MRJobConfig.TaskUserlogLimit, userLogKb);
sleepConf.SetInt(MRJobConfig.TaskLogBackups, 3);
sleepConf.Set(MRJobConfig.MrAmLogLevel, Level.All.ToString());
long amLogKb = 7;
sleepConf.SetLong(MRJobConfig.MrAmLogKb, amLogKb);
sleepConf.SetInt(MRJobConfig.MrAmLogBackups, 7);
sleepJob.SetConf(sleepConf);
Job job = sleepJob.CreateJob(1, 0, 1L, 100, 0L, 0);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.WaitForCompletion(true);
JobId jobId = TypeConverter.ToYarn(job.GetJobID());
ApplicationId appID = jobId.GetAppId();
int pollElapsed = 0;
while (true)
{
Sharpen.Thread.Sleep(1000);
pollElapsed += 1000;
if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps
()[appID].GetState()))
{
break;
}
if (pollElapsed >= 60000)
{
Log.Warn("application did not reach terminal state within 60 seconds");
break;
}
}
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager
().GetRMContext().GetRMApps()[appID].GetState());
// Job finished, verify logs
//
string appIdStr = appID.ToString();
string appIdSuffix = Sharpen.Runtime.Substring(appIdStr, "application_".Length, appIdStr
.Length);
string containerGlob = "container_" + appIdSuffix + "_*_*";
string syslogGlob = appIdStr + Path.Separator + containerGlob + Path.Separator +
TaskLog.LogName.Syslog;
int numAppMasters = 0;
int numMapTasks = 0;
for (int i = 0; i < NumNodeMgrs; i++)
{
Configuration nmConf = mrCluster.GetNodeManager(i).GetConfig();
foreach (string logDir in nmConf.GetTrimmedStrings(YarnConfiguration.NmLogDirs))
{
Path absSyslogGlob = new Path(logDir + Path.Separator + syslogGlob);
Log.Info("Checking for glob: " + absSyslogGlob);
FileStatus[] syslogs = localFs.GlobStatus(absSyslogGlob);
foreach (FileStatus slog in syslogs)
{
bool foundAppMaster = job.IsUber();
Path containerPathComponent = slog.GetPath().GetParent();
if (!foundAppMaster)
{
ContainerId cid = ConverterUtils.ToContainerId(containerPathComponent.GetName());
foundAppMaster = ((cid.GetContainerId() & ContainerId.ContainerIdBitmask) == 1);
}
FileStatus[] sysSiblings = localFs.GlobStatus(new Path(containerPathComponent, TaskLog.LogName
.Syslog + "*"));
// sort to ensure for i > 0 sysSiblings[i] == "syslog.i"
Arrays.Sort(sysSiblings);
if (foundAppMaster)
{
numAppMasters++;
}
else
{
numMapTasks++;
}
if (foundAppMaster)
{
NUnit.Framework.Assert.AreSame("Unexpected number of AM sylog* files", sleepConf.
GetInt(MRJobConfig.MrAmLogBackups, 0) + 1, sysSiblings.Length);
NUnit.Framework.Assert.IsTrue("AM syslog.1 length kb should be >= " + amLogKb, sysSiblings
[1].GetLen() >= amLogKb * 1024);
}
else
{
NUnit.Framework.Assert.AreSame("Unexpected number of MR task sylog* files", sleepConf
.GetInt(MRJobConfig.TaskLogBackups, 0) + 1, sysSiblings.Length);
NUnit.Framework.Assert.IsTrue("MR syslog.1 length kb should be >= " + userLogKb,
sysSiblings[1].GetLen() >= userLogKb * 1024);
}
}
}
}
// Make sure we checked non-empty set
//
NUnit.Framework.Assert.AreEqual("No AppMaster log found!", 1, numAppMasters);
if (sleepConf.GetBoolean(MRJobConfig.JobUbertaskEnable, false))
{
NUnit.Framework.Assert.AreEqual("MapTask log with uber found!", 0, numMapTasks);
}
else
{
NUnit.Framework.Assert.AreEqual("No MapTask log found!", 1, numMapTasks);
}
}
private void ConstructJobReport()
{
report = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <JobReport>();
report.SetJobId(jobId);
report.SetJobState(JobState.ValueOf(jobInfo.GetJobStatus()));
report.SetSubmitTime(jobInfo.GetSubmitTime());
report.SetStartTime(jobInfo.GetLaunchTime());
report.SetFinishTime(jobInfo.GetFinishTime());
report.SetJobName(jobInfo.GetJobname());
report.SetUser(jobInfo.GetUsername());
if (GetTotalMaps() == 0)
{
report.SetMapProgress(1.0f);
}
else
{
report.SetMapProgress((float)GetCompletedMaps() / GetTotalMaps());
}
if (GetTotalReduces() == 0)
{
report.SetReduceProgress(1.0f);
}
else
{
report.SetReduceProgress((float)GetCompletedReduces() / GetTotalReduces());
}
report.SetJobFile(GetConfFile().ToString());
string historyUrl = "N/A";
try
{
historyUrl = MRWebAppUtil.GetApplicationWebURLOnJHSWithoutScheme(conf, jobId.GetAppId
());
}
catch (UnknownHostException)
{
}
//Ignore.
report.SetTrackingUrl(historyUrl);
report.SetAMInfos(GetAMInfos());
report.SetIsUber(IsUber());
}