コード例 #1
0
 // by being synchronized MultipleOutputTask can be use with a
 // MultithreadedMapper.
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="System.Exception"/>
 private RecordWriter GetRecordWriter(TaskAttemptContext taskContext, string baseFileName
                                      )
 {
     lock (this)
     {
         // look for record-writer in the cache
         RecordWriter writer = recordWriters[baseFileName];
         // If not in cache, create a new one
         if (writer == null)
         {
             // get the record writer from context output format
             FileOutputFormat.SetOutputName(taskContext, baseFileName);
             try
             {
                 writer = ((OutputFormat)ReflectionUtils.NewInstance(taskContext.GetOutputFormatClass
                                                                         (), taskContext.GetConfiguration())).GetRecordWriter(taskContext);
             }
             catch (TypeLoadException e)
             {
                 throw new IOException(e);
             }
             // if counters are enabled, wrap the writer with context
             // to increment counters
             if (countersEnabled)
             {
                 writer = new MultipleOutputs.RecordWriterWithCounter(writer, baseFileName, context
                                                                      );
             }
             // add the record-writer to the cache
             recordWriters[baseFileName] = writer;
         }
         return(writer);
     }
 }
コード例 #2
0
        /// <exception cref="System.Exception"/>
        private void TestMapFileOutputCommitterInternal(int version)
        {
            Job job = Job.GetInstance();

            FileOutputFormat.SetOutputPath(job, outDir);
            Configuration conf = job.GetConfiguration();

            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            JobContext          jContext  = new JobContextImpl(conf, taskID.GetJobID());
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

            // setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            MapFileOutputFormat theOutputFormat = new MapFileOutputFormat();
            RecordWriter        theRecordWriter = theOutputFormat.GetRecordWriter(tContext);

            WriteMapFileOutput(theRecordWriter, tContext);
            // do commit
            committer.CommitTask(tContext);
            committer.CommitJob(jContext);
            // validate output
            ValidateMapFileOutputContent(FileSystem.Get(job.GetConfiguration()), outDir);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
コード例 #3
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCheckOutputSpecsException()
        {
            Job        job    = Job.GetInstance();
            Path       outDir = new Path(Runtime.GetProperty("test.build.data", "/tmp"), "output");
            FileSystem fs     = outDir.GetFileSystem(new Configuration());

            // Create the output dir so it already exists and set it for the job
            fs.Mkdirs(outDir);
            FileOutputFormat.SetOutputPath(job, outDir);
            // We don't need a "full" implementation of FileOutputFormat for this test
            FileOutputFormat fof = new _FileOutputFormat_54();

            try
            {
                try
                {
                    // This should throw a FileAlreadyExistsException because the outputDir
                    // already exists
                    fof.CheckOutputSpecs(job);
                    Fail("Should have thrown a FileAlreadyExistsException");
                }
                catch (FileAlreadyExistsException)
                {
                }
            }
            finally
            {
                // correct behavior
                // Cleanup
                if (fs.Exists(outDir))
                {
                    fs.Delete(outDir, true);
                }
            }
        }
コード例 #4
0
            /// <exception cref="System.IO.IOException"/>
            public override void CleanupJob(JobContext context)
            {
                System.Console.Error.WriteLine("---- HERE ----");
                Path       outputPath = FileOutputFormat.GetOutputPath(context);
                FileSystem fs         = outputPath.GetFileSystem(context.GetConfiguration());

                fs.Create(new Path(outputPath, CustomCleanupFileName)).Close();
            }
コード例 #5
0
            /// <exception cref="System.IO.IOException"/>
            public override Path GetDefaultWorkFile(TaskAttemptContext context, string extension
                                                    )
            {
                FileOutputCommitter foc = (FileOutputCommitter)this._enclosing._enclosing.GetOutputCommitter
                                              (context);

                return(new Path(new Path(foc.GetWorkPath(), TestFileOutputCommitter.SubDir), FileOutputFormat
                                .GetUniqueFile(context, FileOutputFormat.GetOutputName(context), extension)));
            }
コード例 #6
0
            /// <exception cref="System.IO.IOException"/>
            public override void AbortJob(JobContext context, JobStatus.State state)
            {
                Path       outputPath = FileOutputFormat.GetOutputPath(context);
                FileSystem fs         = outputPath.GetFileSystem(context.GetConfiguration());
                string     fileName   = (state.Equals(JobStatus.State.Failed)) ? AbortFailedFileName :
                                        AbortKilledFileName;

                fs.Create(new Path(outputPath, fileName)).Close();
            }
コード例 #7
0
        /// <exception cref="System.Exception"/>
        private void TestConcurrentCommitTaskWithSubDir(int version)
        {
            Job job = Job.GetInstance();

            FileOutputFormat.SetOutputPath(job, outDir);
            Configuration conf = job.GetConfiguration();

            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            conf.SetClass("fs.file.impl", typeof(TestFileOutputCommitter.RLFS), typeof(FileSystem
                                                                                       ));
            FileSystem.CloseAll();
            JobContext          jContext    = new JobContextImpl(conf, taskID.GetJobID());
            FileOutputCommitter amCommitter = new FileOutputCommitter(outDir, jContext);

            amCommitter.SetupJob(jContext);
            TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
            taCtx[0] = new TaskAttemptContextImpl(conf, taskID);
            taCtx[1] = new TaskAttemptContextImpl(conf, taskID1);
            TextOutputFormat[] tof = new TextOutputFormat[2];
            for (int i = 0; i < tof.Length; i++)
            {
                tof[i] = new _TextOutputFormat_508(this);
            }
            ExecutorService executor = Executors.NewFixedThreadPool(2);

            try
            {
                for (int i_1 = 0; i_1 < taCtx.Length; i_1++)
                {
                    int taskIdx = i_1;
                    executor.Submit(new _Callable_524(this, tof, taskIdx, taCtx));
                }
            }
            finally
            {
                executor.Shutdown();
                while (!executor.AwaitTermination(1, TimeUnit.Seconds))
                {
                    Log.Info("Awaiting thread termination!");
                }
            }
            amCommitter.CommitJob(jContext);
            RawLocalFileSystem lfs = new RawLocalFileSystem();

            lfs.SetConf(conf);
            NUnit.Framework.Assert.IsFalse("Must not end up with sub_dir/sub_dir", lfs.Exists
                                               (new Path(OutSubDir, SubDir)));
            // validate output
            ValidateContent(OutSubDir);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
コード例 #8
0
        /// <exception cref="System.Exception"/>
        public virtual void TestSetOutputPathException()
        {
            Job job = Job.GetInstance();

            try
            {
                // Give it an invalid filesystem so it'll throw an exception
                FileOutputFormat.SetOutputPath(job, new Path("foo:///bar"));
                Fail("Should have thrown a RuntimeException with an IOException inside");
            }
            catch (RuntimeException re)
            {
                NUnit.Framework.Assert.IsTrue(re.InnerException is IOException);
            }
        }
コード例 #9
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestcheckOutputSpecsForbidRecordCompression()
        {
            Job        job       = Job.GetInstance();
            FileSystem fs        = FileSystem.GetLocal(job.GetConfiguration());
            Path       outputdir = new Path(Runtime.GetProperty("test.build.data", "/tmp") + "/output"
                                            );

            fs.Delete(outputdir, true);
            // Without outputpath, FileOutputFormat.checkoutputspecs will throw
            // InvalidJobConfException
            FileOutputFormat.SetOutputPath(job, outputdir);
            // SequenceFileAsBinaryOutputFormat doesn't support record compression
            // It should throw an exception when checked by checkOutputSpecs
            SequenceFileAsBinaryOutputFormat.SetCompressOutput(job, true);
            SequenceFileAsBinaryOutputFormat.SetOutputCompressionType(job, SequenceFile.CompressionType
                                                                      .Block);
            try
            {
                new SequenceFileAsBinaryOutputFormat().CheckOutputSpecs(job);
            }
            catch (Exception e)
            {
                Fail("Block compression should be allowed for " + "SequenceFileAsBinaryOutputFormat:Caught "
                     + e.GetType().FullName);
            }
            SequenceFileAsBinaryOutputFormat.SetOutputCompressionType(job, SequenceFile.CompressionType
                                                                      .Record);
            try
            {
                new SequenceFileAsBinaryOutputFormat().CheckOutputSpecs(job);
                Fail("Record compression should not be allowed for " + "SequenceFileAsBinaryOutputFormat"
                     );
            }
            catch (InvalidJobConfException)
            {
            }
            catch (Exception e)
            {
                // expected
                Fail("Expected " + typeof(InvalidJobConfException).FullName + "but caught " + e.GetType
                         ().FullName);
            }
        }
コード例 #10
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestInvalidVersionNumber()
        {
            Job job = Job.GetInstance();

            FileOutputFormat.SetOutputPath(job, outDir);
            Configuration conf = job.GetConfiguration();

            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, 3);
            TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);

            try
            {
                new FileOutputCommitter(outDir, tContext);
                Fail("should've thrown an exception!");
            }
            catch (IOException)
            {
            }
        }
コード例 #11
0
        /// <exception cref="System.Exception"/>
        public virtual void TestEmptyOutput()
        {
            Job job = Job.GetInstance();

            FileOutputFormat.SetOutputPath(job, outDir);
            Configuration conf = job.GetConfiguration();

            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            JobContext          jContext  = new JobContextImpl(conf, taskID.GetJobID());
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

            // setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // Do not write any output
            // do commit
            committer.CommitTask(tContext);
            committer.CommitJob(jContext);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
コード例 #12
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCommitter()
        {
            Job job = Job.GetInstance();

            FileOutputFormat.SetOutputPath(job, outDir);
            Configuration conf = job.GetConfiguration();

            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            JobContext          jContext  = new JobContextImpl(conf, taskID.GetJobID());
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

            // setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(tContext);

            WriteOutput(theRecordWriter, tContext);
            // do commit
            committer.CommitTask(tContext);
            committer.CommitJob(jContext);
            // validate output
            FilePath      expectedFile   = new FilePath(new Path(outDir, partFile).ToString());
            StringBuilder expectedOutput = new StringBuilder();

            expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n");
            expectedOutput.Append(val1).Append("\n");
            expectedOutput.Append(val2).Append("\n");
            expectedOutput.Append(key2).Append("\n");
            expectedOutput.Append(key1).Append("\n");
            expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n");
            string output = UtilsForTests.Slurp(expectedFile);

            NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
コード例 #13
0
        //test passed
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        private void TestAbortInternal(int version)
        {
            Job job = Job.GetInstance();

            FileOutputFormat.SetOutputPath(job, outDir);
            Configuration conf = job.GetConfiguration();

            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            JobContext          jContext  = new JobContextImpl(conf, taskID.GetJobID());
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(tContext);

            WriteOutput(theRecordWriter, tContext);
            // do abort
            committer.AbortTask(tContext);
            FilePath expectedFile = new FilePath(new Path(committer.GetWorkPath(), partFile).
                                                 ToString());

            NUnit.Framework.Assert.IsFalse("task temp dir still exists", expectedFile.Exists(
                                               ));
            committer.AbortJob(jContext, JobStatus.State.Failed);
            expectedFile = new FilePath(new Path(outDir, FileOutputCommitter.PendingDirName).
                                        ToString());
            NUnit.Framework.Assert.IsFalse("job temp dir still exists", expectedFile.Exists()
                                           );
            NUnit.Framework.Assert.AreEqual("Output directory not empty", 0, new FilePath(outDir
                                                                                          .ToString()).ListFiles().Length);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
コード例 #14
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        private void TestFailAbortInternal(int version)
        {
            Job           job  = Job.GetInstance();
            Configuration conf = job.GetConfiguration();

            conf.Set(FileSystem.FsDefaultNameKey, "faildel:///");
            conf.SetClass("fs.faildel.impl", typeof(TestFileOutputCommitter.FakeFileSystem),
                          typeof(FileSystem));
            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            conf.SetInt(MRJobConfig.ApplicationAttemptId, 1);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
            FileOutputFormat.SetOutputPath(job, outDir);
            JobContext          jContext  = new JobContextImpl(conf, taskID.GetJobID());
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

            // do setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat();
            RecordWriter <object, object>     theRecordWriter = theOutputFormat.GetRecordWriter(tContext
                                                                                                );

            WriteOutput(theRecordWriter, tContext);
            // do abort
            Exception th = null;

            try
            {
                committer.AbortTask(tContext);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            Path     jtd          = committer.GetJobAttemptPath(jContext);
            FilePath jobTmpDir    = new FilePath(jtd.ToUri().GetPath());
            Path     ttd          = committer.GetTaskAttemptPath(tContext);
            FilePath taskTmpDir   = new FilePath(ttd.ToUri().GetPath());
            FilePath expectedFile = new FilePath(taskTmpDir, partFile);

            NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists
                                              ());
            th = null;
            try
            {
                committer.AbortJob(jContext, JobStatus.State.Failed);
            }
            catch (IOException ie)
            {
                th = ie;
            }
            NUnit.Framework.Assert.IsNotNull(th);
            NUnit.Framework.Assert.IsTrue(th is IOException);
            NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
            NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists());
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }
コード例 #15
0
        /// <exception cref="System.Exception"/>
        protected internal virtual void _testMOWithJavaSerialization(bool withCounters)
        {
            string        input = "a\nb\nc\nd\ne\nc\nd\ne";
            Configuration conf  = CreateJobConf();

            conf.Set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
                     + "org.apache.hadoop.io.serializer.WritableSerialization");
            Job job = MapReduceTestUtil.CreateJob(conf, InDir, OutDir, 2, 1, input);

            job.SetJobName("mo");
            MultipleOutputs.AddNamedOutput(job, Text, typeof(TextOutputFormat), typeof(long),
                                           typeof(string));
            MultipleOutputs.SetCountersEnabled(job, withCounters);
            job.SetSortComparatorClass(typeof(JavaSerializationComparator));
            job.SetMapOutputKeyClass(typeof(long));
            job.SetMapOutputValueClass(typeof(string));
            job.SetOutputKeyClass(typeof(long));
            job.SetOutputValueClass(typeof(string));
            job.SetMapperClass(typeof(TestMRMultipleOutputs.MOJavaSerDeMap));
            job.SetReducerClass(typeof(TestMRMultipleOutputs.MOJavaSerDeReduce));
            job.WaitForCompletion(true);
            // assert number of named output part files
            int        namedOutputCount      = 0;
            int        valueBasedOutputCount = 0;
            FileSystem fs = OutDir.GetFileSystem(conf);

            FileStatus[] statuses = fs.ListStatus(OutDir);
            foreach (FileStatus status in statuses)
            {
                string fileName = status.GetPath().GetName();
                if (fileName.Equals("text-m-00000") || fileName.Equals("text-m-00001") || fileName
                    .Equals("text-r-00000"))
                {
                    namedOutputCount++;
                }
                else
                {
                    if (fileName.Equals("a-r-00000") || fileName.Equals("b-r-00000") || fileName.Equals
                            ("c-r-00000") || fileName.Equals("d-r-00000") || fileName.Equals("e-r-00000"))
                    {
                        valueBasedOutputCount++;
                    }
                }
            }
            NUnit.Framework.Assert.AreEqual(3, namedOutputCount);
            NUnit.Framework.Assert.AreEqual(5, valueBasedOutputCount);
            // assert TextOutputFormat files correctness
            BufferedReader reader = new BufferedReader(new InputStreamReader(fs.Open(new Path
                                                                                         (FileOutputFormat.GetOutputPath(job), "text-r-00000"))));
            int    count = 0;
            string line  = reader.ReadLine();

            while (line != null)
            {
                NUnit.Framework.Assert.IsTrue(line.EndsWith(Text));
                line = reader.ReadLine();
                count++;
            }
            reader.Close();
            NUnit.Framework.Assert.IsFalse(count == 0);
            if (withCounters)
            {
                CounterGroup counters = job.GetCounters().GetGroup(typeof(MultipleOutputs).FullName
                                                                   );
                NUnit.Framework.Assert.AreEqual(6, counters.Size());
                NUnit.Framework.Assert.AreEqual(4, counters.FindCounter(Text).GetValue());
                NUnit.Framework.Assert.AreEqual(2, counters.FindCounter("a").GetValue());
                NUnit.Framework.Assert.AreEqual(2, counters.FindCounter("b").GetValue());
                NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("c").GetValue());
                NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("d").GetValue());
                NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("e").GetValue());
            }
        }
コード例 #16
0
        /// <exception cref="System.Exception"/>
        protected internal virtual void _testMultipleOutputs(bool withCounters)
        {
            string        input = "a\nb\nc\nd\ne\nc\nd\ne";
            Configuration conf  = CreateJobConf();
            Job           job   = MapReduceTestUtil.CreateJob(conf, InDir, OutDir, 2, 1, input);

            job.SetJobName("mo");
            MultipleOutputs.AddNamedOutput(job, Text, typeof(TextOutputFormat), typeof(LongWritable
                                                                                       ), typeof(Text));
            MultipleOutputs.AddNamedOutput(job, Sequence, typeof(SequenceFileOutputFormat), typeof(
                                               IntWritable), typeof(Text));
            MultipleOutputs.SetCountersEnabled(job, withCounters);
            job.SetMapperClass(typeof(TestMRMultipleOutputs.MOMap));
            job.SetReducerClass(typeof(TestMRMultipleOutputs.MOReduce));
            job.WaitForCompletion(true);
            // assert number of named output part files
            int        namedOutputCount      = 0;
            int        valueBasedOutputCount = 0;
            FileSystem fs = OutDir.GetFileSystem(conf);

            FileStatus[] statuses = fs.ListStatus(OutDir);
            foreach (FileStatus status in statuses)
            {
                string fileName = status.GetPath().GetName();
                if (fileName.Equals("text-m-00000") || fileName.Equals("text-m-00001") || fileName
                    .Equals("text-r-00000") || fileName.Equals("sequence_A-m-00000") || fileName.Equals
                        ("sequence_A-m-00001") || fileName.Equals("sequence_B-m-00000") || fileName.Equals
                        ("sequence_B-m-00001") || fileName.Equals("sequence_B-r-00000") || fileName.Equals
                        ("sequence_C-r-00000"))
                {
                    namedOutputCount++;
                }
                else
                {
                    if (fileName.Equals("a-r-00000") || fileName.Equals("b-r-00000") || fileName.Equals
                            ("c-r-00000") || fileName.Equals("d-r-00000") || fileName.Equals("e-r-00000"))
                    {
                        valueBasedOutputCount++;
                    }
                }
            }
            NUnit.Framework.Assert.AreEqual(9, namedOutputCount);
            NUnit.Framework.Assert.AreEqual(5, valueBasedOutputCount);
            // assert TextOutputFormat files correctness
            BufferedReader reader = new BufferedReader(new InputStreamReader(fs.Open(new Path
                                                                                         (FileOutputFormat.GetOutputPath(job), "text-r-00000"))));
            int    count = 0;
            string line  = reader.ReadLine();

            while (line != null)
            {
                NUnit.Framework.Assert.IsTrue(line.EndsWith(Text));
                line = reader.ReadLine();
                count++;
            }
            reader.Close();
            NUnit.Framework.Assert.IsFalse(count == 0);
            // assert SequenceOutputFormat files correctness
            SequenceFile.Reader seqReader = new SequenceFile.Reader(fs, new Path(FileOutputFormat
                                                                                 .GetOutputPath(job), "sequence_B-r-00000"), conf);
            NUnit.Framework.Assert.AreEqual(typeof(IntWritable), seqReader.GetKeyClass());
            NUnit.Framework.Assert.AreEqual(typeof(Text), seqReader.GetValueClass());
            count = 0;
            IntWritable key   = new IntWritable();
            Text        value = new Text();

            while (seqReader.Next(key, value))
            {
                NUnit.Framework.Assert.AreEqual(Sequence, value.ToString());
                count++;
            }
            seqReader.Close();
            NUnit.Framework.Assert.IsFalse(count == 0);
            if (withCounters)
            {
                CounterGroup counters = job.GetCounters().GetGroup(typeof(MultipleOutputs).FullName
                                                                   );
                NUnit.Framework.Assert.AreEqual(9, counters.Size());
                NUnit.Framework.Assert.AreEqual(4, counters.FindCounter(Text).GetValue());
                NUnit.Framework.Assert.AreEqual(2, counters.FindCounter(Sequence + "_A").GetValue
                                                    ());
                NUnit.Framework.Assert.AreEqual(4, counters.FindCounter(Sequence + "_B").GetValue
                                                    ());
                NUnit.Framework.Assert.AreEqual(2, counters.FindCounter(Sequence + "_C").GetValue
                                                    ());
                NUnit.Framework.Assert.AreEqual(2, counters.FindCounter("a").GetValue());
                NUnit.Framework.Assert.AreEqual(2, counters.FindCounter("b").GetValue());
                NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("c").GetValue());
                NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("d").GetValue());
                NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("e").GetValue());
            }
        }
コード例 #17
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        public virtual void TestBinary()
        {
            Configuration conf   = new Configuration();
            Job           job    = Job.GetInstance(conf);
            Path          outdir = new Path(Runtime.GetProperty("test.build.data", "/tmp"), "outseq");
            Random        r      = new Random();
            long          seed   = r.NextLong();

            r.SetSeed(seed);
            FileOutputFormat.SetOutputPath(job, outdir);
            SequenceFileAsBinaryOutputFormat.SetSequenceFileOutputKeyClass(job, typeof(IntWritable
                                                                                       ));
            SequenceFileAsBinaryOutputFormat.SetSequenceFileOutputValueClass(job, typeof(DoubleWritable
                                                                                         ));
            SequenceFileAsBinaryOutputFormat.SetCompressOutput(job, true);
            SequenceFileAsBinaryOutputFormat.SetOutputCompressionType(job, SequenceFile.CompressionType
                                                                      .Block);
            BytesWritable      bkey    = new BytesWritable();
            BytesWritable      bval    = new BytesWritable();
            TaskAttemptContext context = MapReduceTestUtil.CreateDummyMapTaskAttemptContext(job
                                                                                            .GetConfiguration());
            OutputFormat <BytesWritable, BytesWritable> outputFormat = new SequenceFileAsBinaryOutputFormat
                                                                           ();
            OutputCommitter committer = outputFormat.GetOutputCommitter(context);

            committer.SetupJob(job);
            RecordWriter <BytesWritable, BytesWritable> writer = outputFormat.GetRecordWriter(
                context);
            IntWritable      iwritable = new IntWritable();
            DoubleWritable   dwritable = new DoubleWritable();
            DataOutputBuffer outbuf    = new DataOutputBuffer();

            Log.Info("Creating data by SequenceFileAsBinaryOutputFormat");
            try
            {
                for (int i = 0; i < Records; ++i)
                {
                    iwritable = new IntWritable(r.Next());
                    iwritable.Write(outbuf);
                    bkey.Set(outbuf.GetData(), 0, outbuf.GetLength());
                    outbuf.Reset();
                    dwritable = new DoubleWritable(r.NextDouble());
                    dwritable.Write(outbuf);
                    bval.Set(outbuf.GetData(), 0, outbuf.GetLength());
                    outbuf.Reset();
                    writer.Write(bkey, bval);
                }
            }
            finally
            {
                writer.Close(context);
            }
            committer.CommitTask(context);
            committer.CommitJob(job);
            InputFormat <IntWritable, DoubleWritable> iformat = new SequenceFileInputFormat <IntWritable
                                                                                             , DoubleWritable>();
            int count = 0;

            r.SetSeed(seed);
            SequenceFileInputFormat.SetInputPaths(job, outdir);
            Log.Info("Reading data by SequenceFileInputFormat");
            foreach (InputSplit split in iformat.GetSplits(job))
            {
                RecordReader <IntWritable, DoubleWritable> reader = iformat.CreateRecordReader(split
                                                                                               , context);
                MapContext <IntWritable, DoubleWritable, BytesWritable, BytesWritable> mcontext =
                    new MapContextImpl <IntWritable, DoubleWritable, BytesWritable, BytesWritable>(job
                                                                                                   .GetConfiguration(), context.GetTaskAttemptID(), reader, null, null, MapReduceTestUtil
                                                                                                   .CreateDummyReporter(), split);
                reader.Initialize(split, mcontext);
                try
                {
                    int    sourceInt;
                    double sourceDouble;
                    while (reader.NextKeyValue())
                    {
                        sourceInt    = r.Next();
                        sourceDouble = r.NextDouble();
                        iwritable    = reader.GetCurrentKey();
                        dwritable    = reader.GetCurrentValue();
                        NUnit.Framework.Assert.AreEqual("Keys don't match: " + "*" + iwritable.Get() + ":"
                                                        + sourceInt + "*", sourceInt, iwritable.Get());
                        NUnit.Framework.Assert.IsTrue("Vals don't match: " + "*" + dwritable.Get() + ":"
                                                      + sourceDouble + "*", double.Compare(dwritable.Get(), sourceDouble) == 0);
                        ++count;
                    }
                }
                finally
                {
                    reader.Close();
                }
            }
            NUnit.Framework.Assert.AreEqual("Some records not found", Records, count);
        }
コード例 #18
0
        /// <exception cref="System.Exception"/>
        private void TestRecoveryInternal(int commitVersion, int recoveryVersion)
        {
            Job job = Job.GetInstance();

            FileOutputFormat.SetOutputPath(job, outDir);
            Configuration conf = job.GetConfiguration();

            conf.Set(MRJobConfig.TaskAttemptId, attempt);
            conf.SetInt(MRJobConfig.ApplicationAttemptId, 1);
            conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, commitVersion
                        );
            JobContext          jContext  = new JobContextImpl(conf, taskID.GetJobID());
            TaskAttemptContext  tContext  = new TaskAttemptContextImpl(conf, taskID);
            FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);

            // setup
            committer.SetupJob(jContext);
            committer.SetupTask(tContext);
            // write output
            TextOutputFormat theOutputFormat = new TextOutputFormat();
            RecordWriter     theRecordWriter = theOutputFormat.GetRecordWriter(tContext);

            WriteOutput(theRecordWriter, tContext);
            // do commit
            committer.CommitTask(tContext);
            Path     jobTempDir1 = committer.GetCommittedTaskPath(tContext);
            FilePath jtd         = new FilePath(jobTempDir1.ToUri().GetPath());

            if (commitVersion == 1)
            {
                NUnit.Framework.Assert.IsTrue("Version 1 commits to temporary dir " + jtd, jtd.Exists
                                                  ());
                ValidateContent(jtd);
            }
            else
            {
                NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd, jtd.Exists
                                                   ());
            }
            //now while running the second app attempt,
            //recover the task output from first attempt
            Configuration conf2 = job.GetConfiguration();

            conf2.Set(MRJobConfig.TaskAttemptId, attempt);
            conf2.SetInt(MRJobConfig.ApplicationAttemptId, 2);
            conf2.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, recoveryVersion
                         );
            JobContext          jContext2  = new JobContextImpl(conf2, taskID.GetJobID());
            TaskAttemptContext  tContext2  = new TaskAttemptContextImpl(conf2, taskID);
            FileOutputCommitter committer2 = new FileOutputCommitter(outDir, tContext2);

            committer2.SetupJob(tContext2);
            Path     jobTempDir2 = committer2.GetCommittedTaskPath(tContext2);
            FilePath jtd2        = new FilePath(jobTempDir2.ToUri().GetPath());

            committer2.RecoverTask(tContext2);
            if (recoveryVersion == 1)
            {
                NUnit.Framework.Assert.IsTrue("Version 1 recovers to " + jtd2, jtd2.Exists());
                ValidateContent(jtd2);
            }
            else
            {
                NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd2, jtd2.Exists
                                                   ());
                if (commitVersion == 1)
                {
                    NUnit.Framework.Assert.IsTrue("Version 2  recovery moves to output dir from " + jtd
                                                  , jtd.List().Length == 0);
                }
            }
            committer2.CommitJob(jContext2);
            ValidateContent(outDir);
            FileUtil.FullyDelete(new FilePath(outDir.ToString()));
        }