/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Avro.AvroRemoteException"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestJobHistoryData()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
SleepJob sleepJob = new SleepJob();
sleepJob.SetConf(mrCluster.GetConfig());
// Job with 3 maps and 2 reduces
Job job = sleepJob.CreateJob(3, 2, 1000, 1, 500, 1);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.WaitForCompletion(true);
Counters counterMR = job.GetCounters();
JobId jobId = TypeConverter.ToYarn(job.GetJobID());
ApplicationId appID = jobId.GetAppId();
int pollElapsed = 0;
while (true)
{
Sharpen.Thread.Sleep(1000);
pollElapsed += 1000;
if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps
()[appID].GetState()))
{
break;
}
if (pollElapsed >= 60000)
{
Log.Warn("application did not reach terminal state within 60 seconds");
break;
}
}
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager
().GetRMContext().GetRMApps()[appID].GetState());
Counters counterHS = job.GetCounters();
//TODO the Assert below worked. need to check
//Should we compare each field or convert to V2 counter and compare
Log.Info("CounterHS " + counterHS);
Log.Info("CounterMR " + counterMR);
NUnit.Framework.Assert.AreEqual(counterHS, counterMR);
HSClientProtocol historyClient = InstantiateHistoryProxy();
GetJobReportRequest gjReq = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetJobReportRequest
>();
gjReq.SetJobId(jobId);
JobReport jobReport = historyClient.GetJobReport(gjReq).GetJobReport();
VerifyJobReport(jobReport, jobId);
}
public virtual void TestRMNMInfo()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
RMContext rmc = mrCluster.GetResourceManager().GetRMContext();
ResourceScheduler rms = mrCluster.GetResourceManager().GetResourceScheduler();
RMNMInfo rmInfo = new RMNMInfo(rmc, rms);
string liveNMs = rmInfo.GetLiveNodeManagers();
ObjectMapper mapper = new ObjectMapper();
JsonNode jn = mapper.ReadTree(liveNMs);
NUnit.Framework.Assert.AreEqual("Unexpected number of live nodes:", Numnodemanagers
, jn.Size());
IEnumerator <JsonNode> it = jn.GetEnumerator();
while (it.HasNext())
{
JsonNode n = it.Next();
NUnit.Framework.Assert.IsNotNull(n.Get("HostName"));
NUnit.Framework.Assert.IsNotNull(n.Get("Rack"));
NUnit.Framework.Assert.IsTrue("Node " + n.Get("NodeId") + " should be RUNNING", n
.Get("State").AsText().Contains("RUNNING"));
NUnit.Framework.Assert.IsNotNull(n.Get("NodeHTTPAddress"));
NUnit.Framework.Assert.IsNotNull(n.Get("LastHealthUpdate"));
NUnit.Framework.Assert.IsNotNull(n.Get("HealthReport"));
NUnit.Framework.Assert.IsNotNull(n.Get("NodeManagerVersion"));
NUnit.Framework.Assert.IsNotNull(n.Get("NumContainers"));
NUnit.Framework.Assert.AreEqual(n.Get("NodeId") + ": Unexpected number of used containers"
, 0, n.Get("NumContainers").AsInt());
NUnit.Framework.Assert.AreEqual(n.Get("NodeId") + ": Unexpected amount of used memory"
, 0, n.Get("UsedMemoryMB").AsInt());
NUnit.Framework.Assert.IsNotNull(n.Get("AvailableMemoryMB"));
}
}
/// <exception cref="System.Exception"/>
private void TestProfilerInternal(bool useDefault)
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
SleepJob sleepJob = new SleepJob();
JobConf sleepConf = new JobConf(mrCluster.GetConfig());
sleepConf.SetProfileEnabled(true);
sleepConf.SetProfileTaskRange(true, ProfiledTaskId.ToString());
sleepConf.SetProfileTaskRange(false, ProfiledTaskId.ToString());
if (!useDefault)
{
// use hprof for map to profile.out
sleepConf.Set(MRJobConfig.TaskMapProfileParams, "-agentlib:hprof=cpu=times,heap=sites,force=n,thread=y,verbose=n,"
+ "file=%s");
// use Xprof for reduce to stdout
sleepConf.Set(MRJobConfig.TaskReduceProfileParams, "-Xprof");
}
sleepJob.SetConf(sleepConf);
// 2-map-2-reduce SleepJob
Job job = sleepJob.CreateJob(2, 2, 500, 1, 500, 1);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.WaitForCompletion(true);
JobId jobId = TypeConverter.ToYarn(job.GetJobID());
ApplicationId appID = jobId.GetAppId();
int pollElapsed = 0;
while (true)
{
Sharpen.Thread.Sleep(1000);
pollElapsed += 1000;
if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps
()[appID].GetState()))
{
break;
}
if (pollElapsed >= 60000)
{
Log.Warn("application did not reach terminal state within 60 seconds");
break;
}
}
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager
().GetRMContext().GetRMApps()[appID].GetState());
// Job finished, verify logs
//
Configuration nmConf = mrCluster.GetNodeManager(0).GetConfig();
string appIdStr = appID.ToString();
string appIdSuffix = Sharpen.Runtime.Substring(appIdStr, "application_".Length, appIdStr
.Length);
string containerGlob = "container_" + appIdSuffix + "_*_*";
IDictionary <TaskAttemptID, Path> taLogDirs = new Dictionary <TaskAttemptID, Path>(
);
Sharpen.Pattern taskPattern = Sharpen.Pattern.Compile(".*Task:(attempt_" + appIdSuffix
+ "_[rm]_" + "[0-9]+_[0-9]+).*");
foreach (string logDir in nmConf.GetTrimmedStrings(YarnConfiguration.NmLogDirs))
{
// filter out MRAppMaster and create attemptId->logDir map
//
foreach (FileStatus fileStatus in localFs.GlobStatus(new Path(logDir + Path.Separator
+ appIdStr + Path.Separator + containerGlob + Path.Separator + TaskLog.LogName.
Syslog)))
{
BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(fileStatus
.GetPath())));
string line;
while ((line = br.ReadLine()) != null)
{
Matcher m = taskPattern.Matcher(line);
if (m.Matches())
{
// found Task done message
taLogDirs[TaskAttemptID.ForName(m.Group(1))] = fileStatus.GetPath().GetParent();
break;
}
}
br.Close();
}
}
NUnit.Framework.Assert.AreEqual(4, taLogDirs.Count);
// all 4 attempts found
foreach (KeyValuePair <TaskAttemptID, Path> dirEntry in taLogDirs)
{
TaskAttemptID tid = dirEntry.Key;
Path profilePath = new Path(dirEntry.Value, TaskLog.LogName.Profile.ToString());
Path stdoutPath = new Path(dirEntry.Value, TaskLog.LogName.Stdout.ToString());
if (useDefault || tid.GetTaskType() == TaskType.Map)
{
if (tid.GetTaskID().GetId() == ProfiledTaskId)
{
// verify profile.out
BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(profilePath
)));
string line = br.ReadLine();
NUnit.Framework.Assert.IsTrue("No hprof content found!", line != null && line.StartsWith
("JAVA PROFILE"));
br.Close();
NUnit.Framework.Assert.AreEqual(0L, localFs.GetFileStatus(stdoutPath).GetLen());
}
else
{
NUnit.Framework.Assert.IsFalse("hprof file should not exist", localFs.Exists(profilePath
));
}
}
else
{
NUnit.Framework.Assert.IsFalse("hprof file should not exist", localFs.Exists(profilePath
));
if (tid.GetTaskID().GetId() == ProfiledTaskId)
{
// reducer is profiled with Xprof
BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(stdoutPath
)));
bool flatProfFound = false;
string line;
while ((line = br.ReadLine()) != null)
{
if (line.StartsWith("Flat profile"))
{
flatProfFound = true;
break;
}
}
br.Close();
NUnit.Framework.Assert.IsTrue("Xprof flat profile not found!", flatProfFound);
}
else
{
NUnit.Framework.Assert.AreEqual(0L, localFs.GetFileStatus(stdoutPath).GetLen());
}
}
}
}