/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Avro.AvroRemoteException"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestJobHistoryData()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
SleepJob sleepJob = new SleepJob();
sleepJob.SetConf(mrCluster.GetConfig());
// Job with 3 maps and 2 reduces
Job job = sleepJob.CreateJob(3, 2, 1000, 1, 500, 1);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.WaitForCompletion(true);
Counters counterMR = job.GetCounters();
JobId jobId = TypeConverter.ToYarn(job.GetJobID());
ApplicationId appID = jobId.GetAppId();
int pollElapsed = 0;
while (true)
{
Sharpen.Thread.Sleep(1000);
pollElapsed += 1000;
if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps
()[appID].GetState()))
{
break;
}
if (pollElapsed >= 60000)
{
Log.Warn("application did not reach terminal state within 60 seconds");
break;
}
}
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager
().GetRMContext().GetRMApps()[appID].GetState());
Counters counterHS = job.GetCounters();
//TODO the Assert below worked. need to check
//Should we compare each field or convert to V2 counter and compare
Log.Info("CounterHS " + counterHS);
Log.Info("CounterMR " + counterMR);
NUnit.Framework.Assert.AreEqual(counterHS, counterMR);
HSClientProtocol historyClient = InstantiateHistoryProxy();
GetJobReportRequest gjReq = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetJobReportRequest
>();
gjReq.SetJobId(jobId);
JobReport jobReport = historyClient.GetJobReport(gjReq).GetJobReport();
VerifyJobReport(jobReport, jobId);
}
public virtual void TestJobWithNonNormalizedCapabilities()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
JobConf jobConf = new JobConf(mrCluster.GetConfig());
jobConf.SetInt("mapreduce.map.memory.mb", 700);
jobConf.SetInt("mapred.reduce.memory.mb", 1500);
SleepJob sleepJob = new SleepJob();
sleepJob.SetConf(jobConf);
Job job = sleepJob.CreateJob(3, 2, 1000, 1, 500, 1);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.Submit();
bool completed = job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job should be completed", completed);
NUnit.Framework.Assert.AreEqual("Job should be finished successfully", JobStatus.State
.Succeeded, job.GetJobState());
}
public static void Setup()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
if (mrCluster == null)
{
mrCluster = new MiniMRYarnCluster(typeof(TestMROldApiJobs).FullName);
mrCluster.Init(new Configuration());
mrCluster.Start();
}
// TestMRJobs is for testing non-uberized operation only; see TestUberAM
// for corresponding uberized tests.
mrCluster.GetConfig().SetBoolean(MRJobConfig.JobUbertaskEnable, false);
// Copy MRAppJar and make it private. TODO: FIXME. This is a hack to
// workaround the absent public discache.
localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), TestMRJobs.AppJar);
localFs.SetPermission(TestMRJobs.AppJar, new FsPermission("700"));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.TypeLoadException"/>
/// <exception cref="System.Exception"/>
private Job RunSpecTest(bool mapspec, bool redspec)
{
Path first = CreateTempFile("specexec_map_input1", "a\nz");
Path secnd = CreateTempFile("specexec_map_input2", "a\nz");
Configuration conf = mrCluster.GetConfig();
conf.SetBoolean(MRJobConfig.MapSpeculative, mapspec);
conf.SetBoolean(MRJobConfig.ReduceSpeculative, redspec);
conf.SetClass(MRJobConfig.MrAmTaskEstimator, typeof(TestSpeculativeExecution.TestSpecEstimator
), typeof(TaskRuntimeEstimator));
Job job = Job.GetInstance(conf);
job.SetJarByClass(typeof(TestSpeculativeExecution));
job.SetMapperClass(typeof(TestSpeculativeExecution.SpeculativeMapper));
job.SetReducerClass(typeof(TestSpeculativeExecution.SpeculativeReducer));
job.SetOutputKeyClass(typeof(Text));
job.SetOutputValueClass(typeof(IntWritable));
job.SetNumReduceTasks(2);
FileInputFormat.SetInputPaths(job, first);
FileInputFormat.AddInputPath(job, secnd);
FileOutputFormat.SetOutputPath(job, TestOutDir);
// Delete output directory if it exists.
try
{
localFs.Delete(TestOutDir, true);
}
catch (IOException)
{
}
// ignore
// Creates the Job Configuration
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.SetMaxMapAttempts(2);
job.Submit();
return(job);
}
/// <exception cref="System.Exception"/>
private void TestProfilerInternal(bool useDefault)
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
SleepJob sleepJob = new SleepJob();
JobConf sleepConf = new JobConf(mrCluster.GetConfig());
sleepConf.SetProfileEnabled(true);
sleepConf.SetProfileTaskRange(true, ProfiledTaskId.ToString());
sleepConf.SetProfileTaskRange(false, ProfiledTaskId.ToString());
if (!useDefault)
{
// use hprof for map to profile.out
sleepConf.Set(MRJobConfig.TaskMapProfileParams, "-agentlib:hprof=cpu=times,heap=sites,force=n,thread=y,verbose=n,"
+ "file=%s");
// use Xprof for reduce to stdout
sleepConf.Set(MRJobConfig.TaskReduceProfileParams, "-Xprof");
}
sleepJob.SetConf(sleepConf);
// 2-map-2-reduce SleepJob
Job job = sleepJob.CreateJob(2, 2, 500, 1, 500, 1);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.WaitForCompletion(true);
JobId jobId = TypeConverter.ToYarn(job.GetJobID());
ApplicationId appID = jobId.GetAppId();
int pollElapsed = 0;
while (true)
{
Sharpen.Thread.Sleep(1000);
pollElapsed += 1000;
if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps
()[appID].GetState()))
{
break;
}
if (pollElapsed >= 60000)
{
Log.Warn("application did not reach terminal state within 60 seconds");
break;
}
}
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager
().GetRMContext().GetRMApps()[appID].GetState());
// Job finished, verify logs
//
Configuration nmConf = mrCluster.GetNodeManager(0).GetConfig();
string appIdStr = appID.ToString();
string appIdSuffix = Sharpen.Runtime.Substring(appIdStr, "application_".Length, appIdStr
.Length);
string containerGlob = "container_" + appIdSuffix + "_*_*";
IDictionary <TaskAttemptID, Path> taLogDirs = new Dictionary <TaskAttemptID, Path>(
);
Sharpen.Pattern taskPattern = Sharpen.Pattern.Compile(".*Task:(attempt_" + appIdSuffix
+ "_[rm]_" + "[0-9]+_[0-9]+).*");
foreach (string logDir in nmConf.GetTrimmedStrings(YarnConfiguration.NmLogDirs))
{
// filter out MRAppMaster and create attemptId->logDir map
//
foreach (FileStatus fileStatus in localFs.GlobStatus(new Path(logDir + Path.Separator
+ appIdStr + Path.Separator + containerGlob + Path.Separator + TaskLog.LogName.
Syslog)))
{
BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(fileStatus
.GetPath())));
string line;
while ((line = br.ReadLine()) != null)
{
Matcher m = taskPattern.Matcher(line);
if (m.Matches())
{
// found Task done message
taLogDirs[TaskAttemptID.ForName(m.Group(1))] = fileStatus.GetPath().GetParent();
break;
}
}
br.Close();
}
}
NUnit.Framework.Assert.AreEqual(4, taLogDirs.Count);
// all 4 attempts found
foreach (KeyValuePair <TaskAttemptID, Path> dirEntry in taLogDirs)
{
TaskAttemptID tid = dirEntry.Key;
Path profilePath = new Path(dirEntry.Value, TaskLog.LogName.Profile.ToString());
Path stdoutPath = new Path(dirEntry.Value, TaskLog.LogName.Stdout.ToString());
if (useDefault || tid.GetTaskType() == TaskType.Map)
{
if (tid.GetTaskID().GetId() == ProfiledTaskId)
{
// verify profile.out
BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(profilePath
)));
string line = br.ReadLine();
NUnit.Framework.Assert.IsTrue("No hprof content found!", line != null && line.StartsWith
("JAVA PROFILE"));
br.Close();
NUnit.Framework.Assert.AreEqual(0L, localFs.GetFileStatus(stdoutPath).GetLen());
}
else
{
NUnit.Framework.Assert.IsFalse("hprof file should not exist", localFs.Exists(profilePath
));
}
}
else
{
NUnit.Framework.Assert.IsFalse("hprof file should not exist", localFs.Exists(profilePath
));
if (tid.GetTaskID().GetId() == ProfiledTaskId)
{
// reducer is profiled with Xprof
BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(stdoutPath
)));
bool flatProfFound = false;
string line;
while ((line = br.ReadLine()) != null)
{
if (line.StartsWith("Flat profile"))
{
flatProfFound = true;
break;
}
}
br.Close();
NUnit.Framework.Assert.IsTrue("Xprof flat profile not found!", flatProfFound);
}
else
{
NUnit.Framework.Assert.AreEqual(0L, localFs.GetFileStatus(stdoutPath).GetLen());
}
}
}
}