/// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        /// <exception cref="System.TypeLoadException"/>
        /// <exception cref="Sharpen.URISyntaxException"/>
        private void TestWithConf(Configuration conf)
        {
            // Create a temporary file of length 1.
            Path first = CreateTempFile("distributed.first", "x");
            // Create two jars with a single file inside them.
            Path second = MakeJar(new Path(TestRootDir, "distributed.second.jar"), 2);
            Path third  = MakeJar(new Path(TestRootDir, "distributed.third.jar"), 3);
            Path fourth = MakeJar(new Path(TestRootDir, "distributed.fourth.jar"), 4);
            Job  job    = Job.GetInstance(conf);

            job.SetMapperClass(typeof(TestMRWithDistributedCache.DistributedCacheCheckerMapper
                                      ));
            job.SetReducerClass(typeof(TestMRWithDistributedCache.DistributedCacheCheckerReducer
                                       ));
            job.SetOutputFormatClass(typeof(NullOutputFormat));
            FileInputFormat.SetInputPaths(job, first);
            // Creates the Job Configuration
            job.AddCacheFile(new URI(first.ToUri().ToString() + "#distributed.first.symlink")
                             );
            job.AddFileToClassPath(second);
            job.AddArchiveToClassPath(third);
            job.AddCacheArchive(fourth.ToUri());
            job.SetMaxMapAttempts(1);
            // speed up failures
            job.Submit();
            NUnit.Framework.Assert.IsTrue(job.WaitForCompletion(false));
        }
예제 #2
0
        /// <exception cref="System.Exception"/>
        public virtual void _testDistributedCache(string jobJarPath)
        {
            if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
            {
                Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
                         );
                return;
            }
            // Create a temporary file of length 1.
            Path first = CreateTempFile("distributed.first", "x");
            // Create two jars with a single file inside them.
            Path second = MakeJar(new Path(TestRootDir, "distributed.second.jar"), 2);
            Path third  = MakeJar(new Path(TestRootDir, "distributed.third.jar"), 3);
            Path fourth = MakeJar(new Path(TestRootDir, "distributed.fourth.jar"), 4);
            Job  job    = Job.GetInstance(mrCluster.GetConfig());

            // Set the job jar to a new "dummy" jar so we can check that its extracted
            // properly
            job.SetJar(jobJarPath);
            // Because the job jar is a "dummy" jar, we need to include the jar with
            // DistributedCacheChecker or it won't be able to find it
            Path distributedCacheCheckerJar = new Path(JarFinder.GetJar(typeof(TestMRJobs.DistributedCacheChecker
                                                                               )));

            job.AddFileToClassPath(distributedCacheCheckerJar.MakeQualified(localFs.GetUri(),
                                                                            distributedCacheCheckerJar.GetParent()));
            job.SetMapperClass(typeof(TestMRJobs.DistributedCacheChecker));
            job.SetOutputFormatClass(typeof(NullOutputFormat));
            FileInputFormat.SetInputPaths(job, first);
            // Creates the Job Configuration
            job.AddCacheFile(new URI(first.ToUri().ToString() + "#distributed.first.symlink")
                             );
            job.AddFileToClassPath(second);
            // The AppMaster jar itself
            job.AddFileToClassPath(AppJar.MakeQualified(localFs.GetUri(), AppJar.GetParent())
                                   );
            job.AddArchiveToClassPath(third);
            job.AddCacheArchive(fourth.ToUri());
            job.SetMaxMapAttempts(1);
            // speed up failures
            job.Submit();
            string trackingUrl = job.GetTrackingURL();
            string jobId       = job.GetJobID().ToString();

            NUnit.Framework.Assert.IsTrue(job.WaitForCompletion(false));
            NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID "
                                          + jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf
                                                                                                      ("_")) + "/"));
        }