/// <exception cref="System.IO.IOException"/>
public static MiniMRClientCluster Create(Type caller, string identifier, int noOfNMs
, Configuration conf)
{
if (conf == null)
{
conf = new Configuration();
}
FileSystem fs = FileSystem.Get(conf);
Path testRootDir = new Path("target", identifier + "-tmpDir").MakeQualified(fs);
Path appJar = new Path(testRootDir, "MRAppJar.jar");
// Copy MRAppJar and make it private.
Path appMasterJar = new Path(MiniMRYarnCluster.Appjar);
fs.CopyFromLocalFile(appMasterJar, appJar);
fs.SetPermission(appJar, new FsPermission("744"));
Job job = Job.GetInstance(conf);
job.AddFileToClassPath(appJar);
Path callerJar = new Path(JarFinder.GetJar(caller));
Path remoteCallerJar = new Path(testRootDir, callerJar.GetName());
fs.CopyFromLocalFile(callerJar, remoteCallerJar);
fs.SetPermission(remoteCallerJar, new FsPermission("744"));
job.AddFileToClassPath(remoteCallerJar);
MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(identifier, noOfNMs);
job.GetConfiguration().Set("minimrclientcluster.caller.name", identifier);
job.GetConfiguration().SetInt("minimrclientcluster.nodemanagers.number", noOfNMs);
miniMRYarnCluster.Init(job.GetConfiguration());
miniMRYarnCluster.Start();
return(new MiniMRYarnClusterAdapter(miniMRYarnCluster));
}
/// <exception cref="System.Exception"/>
public virtual void _testDistributedCache(string jobJarPath)
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
// Create a temporary file of length 1.
Path first = CreateTempFile("distributed.first", "x");
// Create two jars with a single file inside them.
Path second = MakeJar(new Path(TestRootDir, "distributed.second.jar"), 2);
Path third = MakeJar(new Path(TestRootDir, "distributed.third.jar"), 3);
Path fourth = MakeJar(new Path(TestRootDir, "distributed.fourth.jar"), 4);
Job job = Job.GetInstance(mrCluster.GetConfig());
// Set the job jar to a new "dummy" jar so we can check that its extracted
// properly
job.SetJar(jobJarPath);
// Because the job jar is a "dummy" jar, we need to include the jar with
// DistributedCacheChecker or it won't be able to find it
Path distributedCacheCheckerJar = new Path(JarFinder.GetJar(typeof(TestMRJobs.DistributedCacheChecker
)));
job.AddFileToClassPath(distributedCacheCheckerJar.MakeQualified(localFs.GetUri(),
distributedCacheCheckerJar.GetParent()));
job.SetMapperClass(typeof(TestMRJobs.DistributedCacheChecker));
job.SetOutputFormatClass(typeof(NullOutputFormat));
FileInputFormat.SetInputPaths(job, first);
// Creates the Job Configuration
job.AddCacheFile(new URI(first.ToUri().ToString() + "#distributed.first.symlink")
);
job.AddFileToClassPath(second);
// The AppMaster jar itself
job.AddFileToClassPath(AppJar.MakeQualified(localFs.GetUri(), AppJar.GetParent())
);
job.AddArchiveToClassPath(third);
job.AddCacheArchive(fourth.ToUri());
job.SetMaxMapAttempts(1);
// speed up failures
job.Submit();
string trackingUrl = job.GetTrackingURL();
string jobId = job.GetJobID().ToString();
NUnit.Framework.Assert.IsTrue(job.WaitForCompletion(false));
NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID "
+ jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf
("_")) + "/"));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
private void TestWithConf(Configuration conf)
{
// Create a temporary file of length 1.
Path first = CreateTempFile("distributed.first", "x");
// Create two jars with a single file inside them.
Path second = MakeJar(new Path(TestRootDir, "distributed.second.jar"), 2);
Path third = MakeJar(new Path(TestRootDir, "distributed.third.jar"), 3);
Path fourth = MakeJar(new Path(TestRootDir, "distributed.fourth.jar"), 4);
Job job = Job.GetInstance(conf);
job.SetMapperClass(typeof(TestMRWithDistributedCache.DistributedCacheCheckerMapper
));
job.SetReducerClass(typeof(TestMRWithDistributedCache.DistributedCacheCheckerReducer
));
job.SetOutputFormatClass(typeof(NullOutputFormat));
FileInputFormat.SetInputPaths(job, first);
// Creates the Job Configuration
job.AddCacheFile(new URI(first.ToUri().ToString() + "#distributed.first.symlink")
);
job.AddFileToClassPath(second);
job.AddArchiveToClassPath(third);
job.AddCacheArchive(fourth.ToUri());
job.SetMaxMapAttempts(1);
// speed up failures
job.Submit();
NUnit.Framework.Assert.IsTrue(job.WaitForCompletion(false));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
protected internal virtual Job RunFailingMapperJob()
{
Configuration myConf = new Configuration(mrCluster.GetConfig());
myConf.SetInt(MRJobConfig.NumMaps, 1);
myConf.SetInt(MRJobConfig.MapMaxAttempts, 2);
//reduce the number of attempts
Job job = Job.GetInstance(myConf);
job.SetJarByClass(typeof(FailingMapper));
job.SetJobName("failmapper");
job.SetOutputKeyClass(typeof(Text));
job.SetOutputValueClass(typeof(Text));
job.SetInputFormatClass(typeof(RandomTextWriterJob.RandomInputFormat));
job.SetOutputFormatClass(typeof(TextOutputFormat));
job.SetMapperClass(typeof(FailingMapper));
job.SetNumReduceTasks(0);
FileOutputFormat.SetOutputPath(job, new Path(OutputRootDir, "failmapper-output"));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.Submit();
string trackingUrl = job.GetTrackingURL();
string jobId = job.GetJobID().ToString();
bool succeeded = job.WaitForCompletion(true);
NUnit.Framework.Assert.IsFalse(succeeded);
NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID "
+ jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf
("_")) + "/"));
return(job);
}
public virtual void TestJobWithNonNormalizedCapabilities()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
JobConf jobConf = new JobConf(mrCluster.GetConfig());
jobConf.SetInt("mapreduce.map.memory.mb", 700);
jobConf.SetInt("mapred.reduce.memory.mb", 1500);
SleepJob sleepJob = new SleepJob();
sleepJob.SetConf(jobConf);
Job job = sleepJob.CreateJob(3, 2, 1000, 1, 500, 1);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.Submit();
bool completed = job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job should be completed", completed);
NUnit.Framework.Assert.AreEqual("Job should be finished successfully", JobStatus.State
.Succeeded, job.GetJobState());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
private void TestJobClassloader(bool useCustomClasses)
{
Log.Info("\n\n\nStarting testJobClassloader()" + " useCustomClasses=" + useCustomClasses
);
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
Configuration sleepConf = new Configuration(mrCluster.GetConfig());
// set master address to local to test that local mode applied iff framework == local
sleepConf.Set(MRConfig.MasterAddress, "local");
sleepConf.SetBoolean(MRJobConfig.MapreduceJobClassloader, true);
if (useCustomClasses)
{
// to test AM loading user classes such as output format class, we want
// to blacklist them from the system classes (they need to be prepended
// as the first match wins)
string systemClasses = ApplicationClassLoader.SystemClassesDefault;
// exclude the custom classes from system classes
systemClasses = "-" + typeof(TestMRJobs.CustomOutputFormat).FullName + ",-" + typeof(
TestMRJobs.CustomSpeculator).FullName + "," + systemClasses;
sleepConf.Set(MRJobConfig.MapreduceJobClassloaderSystemClasses, systemClasses);
}
sleepConf.Set(MRJobConfig.IoSortMb, TestIoSortMb);
sleepConf.Set(MRJobConfig.MrAmLogLevel, Level.All.ToString());
sleepConf.Set(MRJobConfig.MapLogLevel, Level.All.ToString());
sleepConf.Set(MRJobConfig.ReduceLogLevel, Level.All.ToString());
sleepConf.Set(MRJobConfig.MapJavaOpts, "-verbose:class");
SleepJob sleepJob = new SleepJob();
sleepJob.SetConf(sleepConf);
Job job = sleepJob.CreateJob(1, 1, 10, 1, 10, 1);
job.SetMapperClass(typeof(TestMRJobs.ConfVerificationMapper));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.SetJarByClass(typeof(SleepJob));
job.SetMaxMapAttempts(1);
// speed up failures
if (useCustomClasses)
{
// set custom output format class and speculator class
job.SetOutputFormatClass(typeof(TestMRJobs.CustomOutputFormat));
Configuration jobConf = job.GetConfiguration();
jobConf.SetClass(MRJobConfig.MrAmJobSpeculator, typeof(TestMRJobs.CustomSpeculator
), typeof(Speculator));
// speculation needs to be enabled for the speculator to be loaded
jobConf.SetBoolean(MRJobConfig.MapSpeculative, true);
}
job.Submit();
bool succeeded = job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job status: " + job.GetStatus().GetFailureInfo(),
succeeded);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Avro.AvroRemoteException"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestJobHistoryData()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
SleepJob sleepJob = new SleepJob();
sleepJob.SetConf(mrCluster.GetConfig());
// Job with 3 maps and 2 reduces
Job job = sleepJob.CreateJob(3, 2, 1000, 1, 500, 1);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.WaitForCompletion(true);
Counters counterMR = job.GetCounters();
JobId jobId = TypeConverter.ToYarn(job.GetJobID());
ApplicationId appID = jobId.GetAppId();
int pollElapsed = 0;
while (true)
{
Sharpen.Thread.Sleep(1000);
pollElapsed += 1000;
if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps
()[appID].GetState()))
{
break;
}
if (pollElapsed >= 60000)
{
Log.Warn("application did not reach terminal state within 60 seconds");
break;
}
}
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager
().GetRMContext().GetRMApps()[appID].GetState());
Counters counterHS = job.GetCounters();
//TODO the Assert below worked. need to check
//Should we compare each field or convert to V2 counter and compare
Log.Info("CounterHS " + counterHS);
Log.Info("CounterMR " + counterMR);
NUnit.Framework.Assert.AreEqual(counterHS, counterMR);
HSClientProtocol historyClient = InstantiateHistoryProxy();
GetJobReportRequest gjReq = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetJobReportRequest
>();
gjReq.SetJobId(jobId);
JobReport jobReport = historyClient.GetJobReport(gjReq).GetJobReport();
VerifyJobReport(jobReport, jobId);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestRandomWriter()
{
Log.Info("\n\n\nStarting testRandomWriter().");
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
RandomTextWriterJob randomWriterJob = new RandomTextWriterJob();
mrCluster.GetConfig().Set(RandomTextWriterJob.TotalBytes, "3072");
mrCluster.GetConfig().Set(RandomTextWriterJob.BytesPerMap, "1024");
Job job = randomWriterJob.CreateJob(mrCluster.GetConfig());
Path outputDir = new Path(OutputRootDir, "random-output");
FileOutputFormat.SetOutputPath(job, outputDir);
job.SetSpeculativeExecution(false);
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.SetJarByClass(typeof(RandomTextWriterJob));
job.SetMaxMapAttempts(1);
// speed up failures
job.Submit();
string trackingUrl = job.GetTrackingURL();
string jobId = job.GetJobID().ToString();
bool succeeded = job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue(succeeded);
NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, job.GetJobState());
NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID "
+ jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf
("_")) + "/"));
// Make sure there are three files in the output-dir
RemoteIterator <FileStatus> iterator = FileContext.GetFileContext(mrCluster.GetConfig
()).ListStatus(outputDir);
int count = 0;
while (iterator.HasNext())
{
FileStatus file = iterator.Next();
if (!file.GetPath().GetName().Equals(FileOutputCommitter.SucceededFileName))
{
count++;
}
}
NUnit.Framework.Assert.AreEqual("Number of part files is wrong!", 3, count);
VerifyRandomWriterCounters(job);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
internal virtual void RunTestTaskEnv(JobConf conf, Path inDir, Path outDir, bool
oldConfigs)
{
string input = "The input";
Configure(conf, inDir, outDir, input, typeof(TestMiniMRChildTask.EnvCheckMapper),
typeof(TestMiniMRChildTask.EnvCheckReducer));
// test
// - new SET of new var (MY_PATH)
// - set of old var (LANG)
// - append to an old var from modified env (LD_LIBRARY_PATH)
// - append to an old var from tt's env (PATH)
// - append to a new var (NEW_PATH)
string mapTaskEnvKey = JobConf.MapredMapTaskEnv;
string reduceTaskEnvKey = JobConf.MapredMapTaskEnv;
string mapTaskJavaOptsKey = JobConf.MapredMapTaskJavaOpts;
string reduceTaskJavaOptsKey = JobConf.MapredReduceTaskJavaOpts;
string mapTaskJavaOpts = MapOptsVal;
string reduceTaskJavaOpts = ReduceOptsVal;
conf.SetBoolean(OldConfigs, oldConfigs);
if (oldConfigs)
{
mapTaskEnvKey = reduceTaskEnvKey = JobConf.MapredTaskEnv;
mapTaskJavaOptsKey = reduceTaskJavaOptsKey = JobConf.MapredTaskJavaOpts;
mapTaskJavaOpts = reduceTaskJavaOpts = TaskOptsVal;
}
conf.Set(mapTaskEnvKey, Shell.Windows ? "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=%LD_LIBRARY_PATH%;/tmp,"
+ "PATH=%PATH%;/tmp,NEW_PATH=%NEW_PATH%;/tmp" : "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp,"
+ "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp");
conf.Set(reduceTaskEnvKey, Shell.Windows ? "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=%LD_LIBRARY_PATH%;/tmp,"
+ "PATH=%PATH%;/tmp,NEW_PATH=%NEW_PATH%;/tmp" : "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp,"
+ "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp");
conf.Set("path", Runtime.Getenv("PATH"));
conf.Set(mapTaskJavaOptsKey, mapTaskJavaOpts);
conf.Set(reduceTaskJavaOptsKey, reduceTaskJavaOpts);
Job job = Job.GetInstance(conf);
job.AddFileToClassPath(AppJar);
job.SetJarByClass(typeof(TestMiniMRChildTask));
job.SetMaxMapAttempts(1);
// speed up failures
job.WaitForCompletion(true);
bool succeeded = job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("The environment checker job failed.", succeeded);
}
/// <exception cref="System.Exception"/>
private void TestSleepJobInternal(bool useRemoteJar)
{
Log.Info("\n\n\nStarting testSleepJob: useRemoteJar=" + useRemoteJar);
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
Configuration sleepConf = new Configuration(mrCluster.GetConfig());
// set master address to local to test that local mode applied iff framework == local
sleepConf.Set(MRConfig.MasterAddress, "local");
SleepJob sleepJob = new SleepJob();
sleepJob.SetConf(sleepConf);
// job with 3 maps (10s) and numReduces reduces (5s), 1 "record" each:
Job job = sleepJob.CreateJob(3, numSleepReducers, 10000, 1, 5000, 1);
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
if (useRemoteJar)
{
Path localJar = new Path(ClassUtil.FindContainingJar(typeof(SleepJob)));
ConfigUtil.AddLink(job.GetConfiguration(), "/jobjars", localFs.MakeQualified(localJar
.GetParent()).ToUri());
job.SetJar("viewfs:///jobjars/" + localJar.GetName());
}
else
{
job.SetJarByClass(typeof(SleepJob));
}
job.SetMaxMapAttempts(1);
// speed up failures
job.Submit();
string trackingUrl = job.GetTrackingURL();
string jobId = job.GetJobID().ToString();
bool succeeded = job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue(succeeded);
NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, job.GetJobState());
NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID "
+ jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf
("_")) + "/"));
VerifySleepJobCounters(job);
VerifyTaskProgress(job);
}
/// <summary>Launch tests</summary>
/// <param name="conf">Configuration of the mapreduce job.</param>
/// <param name="inDir">input path</param>
/// <param name="outDir">output path</param>
/// <param name="input">Input text</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void LaunchTest(JobConf conf, Path inDir, Path outDir, string input
)
{
FileSystem outFs = outDir.GetFileSystem(conf);
// Launch job with default option for temp dir.
// i.e. temp dir is ./tmp
Job job = Job.GetInstance(conf);
job.AddFileToClassPath(AppJar);
job.SetJarByClass(typeof(TestMiniMRChildTask));
job.SetMaxMapAttempts(1);
// speed up failures
job.WaitForCompletion(true);
bool succeeded = job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue(succeeded);
outFs.Delete(outDir, true);
}
/// <exception cref="System.Exception"/>
public Void Run()
{
SleepJob sleepJob = new SleepJob();
sleepJob.SetConf(TestMRJobs.mrCluster.GetConfig());
Job job = sleepJob.CreateJob(3, 0, 10000, 1, 0, 0);
// //Job with reduces
// Job job = sleepJob.createJob(3, 2, 10000, 1, 10000, 1);
job.AddFileToClassPath(TestMRJobs.AppJar);
// The AppMaster jar itself.
job.Submit();
string trackingUrl = job.GetTrackingURL();
string jobId = job.GetJobID().ToString();
job.WaitForCompletion(true);
NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, job.GetJobState());
NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID "
+ jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf
("_")) + "/"));
return(null);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.TypeLoadException"/>
/// <exception cref="System.Exception"/>
private Job RunSpecTest(bool mapspec, bool redspec)
{
Path first = CreateTempFile("specexec_map_input1", "a\nz");
Path secnd = CreateTempFile("specexec_map_input2", "a\nz");
Configuration conf = mrCluster.GetConfig();
conf.SetBoolean(MRJobConfig.MapSpeculative, mapspec);
conf.SetBoolean(MRJobConfig.ReduceSpeculative, redspec);
conf.SetClass(MRJobConfig.MrAmTaskEstimator, typeof(TestSpeculativeExecution.TestSpecEstimator
), typeof(TaskRuntimeEstimator));
Job job = Job.GetInstance(conf);
job.SetJarByClass(typeof(TestSpeculativeExecution));
job.SetMapperClass(typeof(TestSpeculativeExecution.SpeculativeMapper));
job.SetReducerClass(typeof(TestSpeculativeExecution.SpeculativeReducer));
job.SetOutputKeyClass(typeof(Text));
job.SetOutputValueClass(typeof(IntWritable));
job.SetNumReduceTasks(2);
FileInputFormat.SetInputPaths(job, first);
FileInputFormat.AddInputPath(job, secnd);
FileOutputFormat.SetOutputPath(job, TestOutDir);
// Delete output directory if it exists.
try
{
localFs.Delete(TestOutDir, true);
}
catch (IOException)
{
}
// ignore
// Creates the Job Configuration
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.SetMaxMapAttempts(2);
job.Submit();
return(job);
}
/// <exception cref="System.Exception"/>
private void TestProfilerInternal(bool useDefault)
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
SleepJob sleepJob = new SleepJob();
JobConf sleepConf = new JobConf(mrCluster.GetConfig());
sleepConf.SetProfileEnabled(true);
sleepConf.SetProfileTaskRange(true, ProfiledTaskId.ToString());
sleepConf.SetProfileTaskRange(false, ProfiledTaskId.ToString());
if (!useDefault)
{
// use hprof for map to profile.out
sleepConf.Set(MRJobConfig.TaskMapProfileParams, "-agentlib:hprof=cpu=times,heap=sites,force=n,thread=y,verbose=n,"
+ "file=%s");
// use Xprof for reduce to stdout
sleepConf.Set(MRJobConfig.TaskReduceProfileParams, "-Xprof");
}
sleepJob.SetConf(sleepConf);
// 2-map-2-reduce SleepJob
Job job = sleepJob.CreateJob(2, 2, 500, 1, 500, 1);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.WaitForCompletion(true);
JobId jobId = TypeConverter.ToYarn(job.GetJobID());
ApplicationId appID = jobId.GetAppId();
int pollElapsed = 0;
while (true)
{
Sharpen.Thread.Sleep(1000);
pollElapsed += 1000;
if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps
()[appID].GetState()))
{
break;
}
if (pollElapsed >= 60000)
{
Log.Warn("application did not reach terminal state within 60 seconds");
break;
}
}
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager
().GetRMContext().GetRMApps()[appID].GetState());
// Job finished, verify logs
//
Configuration nmConf = mrCluster.GetNodeManager(0).GetConfig();
string appIdStr = appID.ToString();
string appIdSuffix = Sharpen.Runtime.Substring(appIdStr, "application_".Length, appIdStr
.Length);
string containerGlob = "container_" + appIdSuffix + "_*_*";
IDictionary <TaskAttemptID, Path> taLogDirs = new Dictionary <TaskAttemptID, Path>(
);
Sharpen.Pattern taskPattern = Sharpen.Pattern.Compile(".*Task:(attempt_" + appIdSuffix
+ "_[rm]_" + "[0-9]+_[0-9]+).*");
foreach (string logDir in nmConf.GetTrimmedStrings(YarnConfiguration.NmLogDirs))
{
// filter out MRAppMaster and create attemptId->logDir map
//
foreach (FileStatus fileStatus in localFs.GlobStatus(new Path(logDir + Path.Separator
+ appIdStr + Path.Separator + containerGlob + Path.Separator + TaskLog.LogName.
Syslog)))
{
BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(fileStatus
.GetPath())));
string line;
while ((line = br.ReadLine()) != null)
{
Matcher m = taskPattern.Matcher(line);
if (m.Matches())
{
// found Task done message
taLogDirs[TaskAttemptID.ForName(m.Group(1))] = fileStatus.GetPath().GetParent();
break;
}
}
br.Close();
}
}
NUnit.Framework.Assert.AreEqual(4, taLogDirs.Count);
// all 4 attempts found
foreach (KeyValuePair <TaskAttemptID, Path> dirEntry in taLogDirs)
{
TaskAttemptID tid = dirEntry.Key;
Path profilePath = new Path(dirEntry.Value, TaskLog.LogName.Profile.ToString());
Path stdoutPath = new Path(dirEntry.Value, TaskLog.LogName.Stdout.ToString());
if (useDefault || tid.GetTaskType() == TaskType.Map)
{
if (tid.GetTaskID().GetId() == ProfiledTaskId)
{
// verify profile.out
BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(profilePath
)));
string line = br.ReadLine();
NUnit.Framework.Assert.IsTrue("No hprof content found!", line != null && line.StartsWith
("JAVA PROFILE"));
br.Close();
NUnit.Framework.Assert.AreEqual(0L, localFs.GetFileStatus(stdoutPath).GetLen());
}
else
{
NUnit.Framework.Assert.IsFalse("hprof file should not exist", localFs.Exists(profilePath
));
}
}
else
{
NUnit.Framework.Assert.IsFalse("hprof file should not exist", localFs.Exists(profilePath
));
if (tid.GetTaskID().GetId() == ProfiledTaskId)
{
// reducer is profiled with Xprof
BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(stdoutPath
)));
bool flatProfFound = false;
string line;
while ((line = br.ReadLine()) != null)
{
if (line.StartsWith("Flat profile"))
{
flatProfFound = true;
break;
}
}
br.Close();
NUnit.Framework.Assert.IsTrue("Xprof flat profile not found!", flatProfFound);
}
else
{
NUnit.Framework.Assert.AreEqual(0L, localFs.GetFileStatus(stdoutPath).GetLen());
}
}
}
}
// TODO later: add explicit "isUber()" checks of some sort
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestContainerRollingLog()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
SleepJob sleepJob = new SleepJob();
JobConf sleepConf = new JobConf(mrCluster.GetConfig());
sleepConf.Set(MRJobConfig.MapLogLevel, Level.All.ToString());
long userLogKb = 4;
sleepConf.SetLong(MRJobConfig.TaskUserlogLimit, userLogKb);
sleepConf.SetInt(MRJobConfig.TaskLogBackups, 3);
sleepConf.Set(MRJobConfig.MrAmLogLevel, Level.All.ToString());
long amLogKb = 7;
sleepConf.SetLong(MRJobConfig.MrAmLogKb, amLogKb);
sleepConf.SetInt(MRJobConfig.MrAmLogBackups, 7);
sleepJob.SetConf(sleepConf);
Job job = sleepJob.CreateJob(1, 0, 1L, 100, 0L, 0);
job.SetJarByClass(typeof(SleepJob));
job.AddFileToClassPath(AppJar);
// The AppMaster jar itself.
job.WaitForCompletion(true);
JobId jobId = TypeConverter.ToYarn(job.GetJobID());
ApplicationId appID = jobId.GetAppId();
int pollElapsed = 0;
while (true)
{
Sharpen.Thread.Sleep(1000);
pollElapsed += 1000;
if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps
()[appID].GetState()))
{
break;
}
if (pollElapsed >= 60000)
{
Log.Warn("application did not reach terminal state within 60 seconds");
break;
}
}
NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager
().GetRMContext().GetRMApps()[appID].GetState());
// Job finished, verify logs
//
string appIdStr = appID.ToString();
string appIdSuffix = Sharpen.Runtime.Substring(appIdStr, "application_".Length, appIdStr
.Length);
string containerGlob = "container_" + appIdSuffix + "_*_*";
string syslogGlob = appIdStr + Path.Separator + containerGlob + Path.Separator +
TaskLog.LogName.Syslog;
int numAppMasters = 0;
int numMapTasks = 0;
for (int i = 0; i < NumNodeMgrs; i++)
{
Configuration nmConf = mrCluster.GetNodeManager(i).GetConfig();
foreach (string logDir in nmConf.GetTrimmedStrings(YarnConfiguration.NmLogDirs))
{
Path absSyslogGlob = new Path(logDir + Path.Separator + syslogGlob);
Log.Info("Checking for glob: " + absSyslogGlob);
FileStatus[] syslogs = localFs.GlobStatus(absSyslogGlob);
foreach (FileStatus slog in syslogs)
{
bool foundAppMaster = job.IsUber();
Path containerPathComponent = slog.GetPath().GetParent();
if (!foundAppMaster)
{
ContainerId cid = ConverterUtils.ToContainerId(containerPathComponent.GetName());
foundAppMaster = ((cid.GetContainerId() & ContainerId.ContainerIdBitmask) == 1);
}
FileStatus[] sysSiblings = localFs.GlobStatus(new Path(containerPathComponent, TaskLog.LogName
.Syslog + "*"));
// sort to ensure for i > 0 sysSiblings[i] == "syslog.i"
Arrays.Sort(sysSiblings);
if (foundAppMaster)
{
numAppMasters++;
}
else
{
numMapTasks++;
}
if (foundAppMaster)
{
NUnit.Framework.Assert.AreSame("Unexpected number of AM sylog* files", sleepConf.
GetInt(MRJobConfig.MrAmLogBackups, 0) + 1, sysSiblings.Length);
NUnit.Framework.Assert.IsTrue("AM syslog.1 length kb should be >= " + amLogKb, sysSiblings
[1].GetLen() >= amLogKb * 1024);
}
else
{
NUnit.Framework.Assert.AreSame("Unexpected number of MR task sylog* files", sleepConf
.GetInt(MRJobConfig.TaskLogBackups, 0) + 1, sysSiblings.Length);
NUnit.Framework.Assert.IsTrue("MR syslog.1 length kb should be >= " + userLogKb,
sysSiblings[1].GetLen() >= userLogKb * 1024);
}
}
}
}
// Make sure we checked non-empty set
//
NUnit.Framework.Assert.AreEqual("No AppMaster log found!", 1, numAppMasters);
if (sleepConf.GetBoolean(MRJobConfig.JobUbertaskEnable, false))
{
NUnit.Framework.Assert.AreEqual("MapTask log with uber found!", 0, numMapTasks);
}
else
{
NUnit.Framework.Assert.AreEqual("No MapTask log found!", 1, numMapTasks);
}
}
