/// <exception cref="System.IO.IOException"/>
public static MiniMRClientCluster Create(Type caller, string identifier, int noOfNMs
, Configuration conf)
{
if (conf == null)
{
conf = new Configuration();
}
FileSystem fs = FileSystem.Get(conf);
Path testRootDir = new Path("target", identifier + "-tmpDir").MakeQualified(fs);
Path appJar = new Path(testRootDir, "MRAppJar.jar");
// Copy MRAppJar and make it private.
Path appMasterJar = new Path(MiniMRYarnCluster.Appjar);
fs.CopyFromLocalFile(appMasterJar, appJar);
fs.SetPermission(appJar, new FsPermission("744"));
Job job = Job.GetInstance(conf);
job.AddFileToClassPath(appJar);
Path callerJar = new Path(JarFinder.GetJar(caller));
Path remoteCallerJar = new Path(testRootDir, callerJar.GetName());
fs.CopyFromLocalFile(callerJar, remoteCallerJar);
fs.SetPermission(remoteCallerJar, new FsPermission("744"));
job.AddFileToClassPath(remoteCallerJar);
MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(identifier, noOfNMs);
job.GetConfiguration().Set("minimrclientcluster.caller.name", identifier);
job.GetConfiguration().SetInt("minimrclientcluster.nodemanagers.number", noOfNMs);
miniMRYarnCluster.Init(job.GetConfiguration());
miniMRYarnCluster.Start();
return(new MiniMRYarnClusterAdapter(miniMRYarnCluster));
}
/// <exception cref="System.Exception"/>
public virtual void _testDistributedCache(string jobJarPath)
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
// Create a temporary file of length 1.
Path first = CreateTempFile("distributed.first", "x");
// Create two jars with a single file inside them.
Path second = MakeJar(new Path(TestRootDir, "distributed.second.jar"), 2);
Path third = MakeJar(new Path(TestRootDir, "distributed.third.jar"), 3);
Path fourth = MakeJar(new Path(TestRootDir, "distributed.fourth.jar"), 4);
Job job = Job.GetInstance(mrCluster.GetConfig());
// Set the job jar to a new "dummy" jar so we can check that its extracted
// properly
job.SetJar(jobJarPath);
// Because the job jar is a "dummy" jar, we need to include the jar with
// DistributedCacheChecker or it won't be able to find it
Path distributedCacheCheckerJar = new Path(JarFinder.GetJar(typeof(TestMRJobs.DistributedCacheChecker
)));
job.AddFileToClassPath(distributedCacheCheckerJar.MakeQualified(localFs.GetUri(),
distributedCacheCheckerJar.GetParent()));
job.SetMapperClass(typeof(TestMRJobs.DistributedCacheChecker));
job.SetOutputFormatClass(typeof(NullOutputFormat));
FileInputFormat.SetInputPaths(job, first);
// Creates the Job Configuration
job.AddCacheFile(new URI(first.ToUri().ToString() + "#distributed.first.symlink")
);
job.AddFileToClassPath(second);
// The AppMaster jar itself
job.AddFileToClassPath(AppJar.MakeQualified(localFs.GetUri(), AppJar.GetParent())
);
job.AddArchiveToClassPath(third);
job.AddCacheArchive(fourth.ToUri());
job.SetMaxMapAttempts(1);
// speed up failures
job.Submit();
string trackingUrl = job.GetTrackingURL();
string jobId = job.GetJobID().ToString();
NUnit.Framework.Assert.IsTrue(job.WaitForCompletion(false));
NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID "
+ jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf
("_")) + "/"));
}