/// <summary>Launches a MR job and tests the job counters against the expected values.
        ///     </summary>
        /// <param name="testName">The name for the job</param>
        /// <param name="mr">The MR cluster</param>
        /// <param name="fileSys">The FileSystem</param>
        /// <param name="in">Input path</param>
        /// <param name="out">Output path</param>
        /// <param name="numMaps">Number of maps</param>
        /// <param name="otherLocalMaps">Expected value of other local maps</param>
        /// <param name="datalocalMaps">Expected value of data(node) local maps</param>
        /// <param name="racklocalMaps">Expected value of rack local maps</param>
        /// <exception cref="System.IO.IOException"/>
        internal static void LaunchJobAndTestCounters(string jobName, MiniMRCluster mr, FileSystem
                                                      fileSys, Path @in, Path @out, int numMaps, int otherLocalMaps, int dataLocalMaps
                                                      , int rackLocalMaps)
        {
            JobConf jobConf = mr.CreateJobConf();

            if (fileSys.Exists(@out))
            {
                fileSys.Delete(@out, true);
            }
            RunningJob job      = LaunchJob(jobConf, @in, @out, numMaps, jobName);
            Counters   counters = job.GetCounters();

            NUnit.Framework.Assert.AreEqual("Number of local maps", counters.GetCounter(JobCounter
                                                                                        .OtherLocalMaps), otherLocalMaps);
            NUnit.Framework.Assert.AreEqual("Number of Data-local maps", counters.GetCounter(
                                                JobCounter.DataLocalMaps), dataLocalMaps);
            NUnit.Framework.Assert.AreEqual("Number of Rack-local maps", counters.GetCounter(
                                                JobCounter.RackLocalMaps), rackLocalMaps);
            mr.WaitUntilIdle();
            mr.Shutdown();
        }
Exemplo n.º 2
0
        // run a job which gets stuck in mapper and kill it.
        /// <exception cref="System.IO.IOException"/>
        private void TestKilledJob(string fileName, Type committer, string[] exclude)
        {
            JobConf jc     = mr.CreateJobConf();
            Path    outDir = GetNewOutputDir();

            ConfigureJob(jc, "kill job with abort()", 1, 0, outDir);
            // set the job to wait for long
            jc.SetMapperClass(typeof(UtilsForTests.KillMapper));
            jc.SetOutputCommitter(committer);
            JobClient  jobClient = new JobClient(jc);
            RunningJob job       = jobClient.SubmitJob(jc);
            JobID      id        = job.GetID();
            Counters   counters  = job.GetCounters();

            // wait for the map to be launched
            while (true)
            {
                if (counters.GetCounter(JobCounter.TotalLaunchedMaps) == 1)
                {
                    break;
                }
                Log.Info("Waiting for a map task to be launched");
                UtilsForTests.WaitFor(100);
                counters = job.GetCounters();
            }
            job.KillJob();
            // kill the job
            job.WaitForCompletion();
            // wait for the job to complete
            NUnit.Framework.Assert.AreEqual("Job was not killed", JobStatus.Killed, job.GetJobState
                                                ());
            if (fileName != null)
            {
                Path testFile = new Path(outDir, fileName);
                NUnit.Framework.Assert.IsTrue("File " + testFile + " missing for job " + id, fileSys
                                              .Exists(testFile));
            }
            // check if the files from the missing set exists
            foreach (string ex in exclude)
            {
                Path file = new Path(outDir, ex);
                NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for killed job "
                                               + id, fileSys.Exists(file));
            }
        }
Exemplo n.º 3
0
        /// <exception cref="System.Exception"/>
        private long GetTaskCounterUsage(JobClient client, JobID id, int numReports, int
                                         taskId, TaskType type)
        {
            TaskReport[] reports = null;
            if (TaskType.Map.Equals(type))
            {
                reports = client.GetMapTaskReports(id);
            }
            else
            {
                if (TaskType.Reduce.Equals(type))
                {
                    reports = client.GetReduceTaskReports(id);
                }
            }
            NUnit.Framework.Assert.IsNotNull("No reports found for task type '" + type.ToString
                                                 () + "' in job " + id, reports);
            // make sure that the total number of reports match the expected
            NUnit.Framework.Assert.AreEqual("Mismatch in task id", numReports, reports.Length
                                            );
            Counters counters = reports[taskId].GetCounters();

            return(counters.GetCounter(TaskCounter.CommittedHeapBytes));
        }