예제 #1
0
        /// <summary>Test that comparator is defined and configured.</summary>
        /// <exception cref="System.Exception"/>
        public static void TestGetComparator()
        {
            Configuration conf = new Configuration();
            // Without conf.
            WritableComparator frobComparator = WritableComparator.Get(typeof(TestWritable.Frob
                                                                              ));

            System.Diagnostics.Debug.Assert((frobComparator is TestWritable.FrobComparator));
            NUnit.Framework.Assert.IsNotNull(frobComparator.GetConf());
            NUnit.Framework.Assert.IsNull(frobComparator.GetConf().Get(TestConfigParam));
            // With conf.
            conf.Set(TestConfigParam, TestConfigValue);
            frobComparator = WritableComparator.Get(typeof(TestWritable.Frob), conf);
            System.Diagnostics.Debug.Assert((frobComparator is TestWritable.FrobComparator));
            NUnit.Framework.Assert.IsNotNull(frobComparator.GetConf());
            Assert.Equal(conf.Get(TestConfigParam), TestConfigValue);
            // Without conf. should reuse configuration.
            frobComparator = WritableComparator.Get(typeof(TestWritable.Frob));
            System.Diagnostics.Debug.Assert((frobComparator is TestWritable.FrobComparator));
            NUnit.Framework.Assert.IsNotNull(frobComparator.GetConf());
            Assert.Equal(conf.Get(TestConfigParam), TestConfigValue);
            // New conf. should use new configuration.
            frobComparator = WritableComparator.Get(typeof(TestWritable.Frob), new Configuration
                                                        ());
            System.Diagnostics.Debug.Assert((frobComparator is TestWritable.FrobComparator));
            NUnit.Framework.Assert.IsNotNull(frobComparator.GetConf());
            NUnit.Framework.Assert.IsNull(frobComparator.GetConf().Get(TestConfigParam));
        }
예제 #2
0
 public Writer(Configuration conf, FileSystem fs, string dirName, WritableComparator
               comparator, Type valClass, SequenceFile.CompressionType compress, Progressable
               progress)
     : this(conf, new Path(dirName), Comparator(comparator), ValueClass(valClass), Compression
                (compress), Progressable(progress))
 {
 }
예제 #3
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCoding()
        {
            string before = "Bad \t encoding \t testcase";

            Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text(before);
            string after = text.ToString();

            Assert.True(before.Equals(after));
            for (int i = 0; i < NumIterations; i++)
            {
                // generate a random string
                if (i == 0)
                {
                    before = GetLongString();
                }
                else
                {
                    before = GetTestString();
                }
                // test string to utf8
                ByteBuffer bb       = Org.Apache.Hadoop.IO.Text.Encode(before);
                byte[]     utf8Text = ((byte[])bb.Array());
                byte[]     utf8Java = Runtime.GetBytesForString(before, "UTF-8");
                Assert.Equal(0, WritableComparator.CompareBytes(utf8Text, 0, bb
                                                                .Limit(), utf8Java, 0, utf8Java.Length));
                // test utf8 to string
                after = Org.Apache.Hadoop.IO.Text.Decode(utf8Java);
                Assert.True(before.Equals(after));
            }
        }
예제 #4
0
        /// <exception cref="System.IO.IOException"/>
        private void SortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile,
                                      SequenceFile.Metadata metadata)
        {
            fs.Delete(sortedFile, true);
            Log.Info("sorting: " + unsortedFile + " to: " + sortedFile);
            WritableComparator comparator = WritableComparator.Get(typeof(RandomDatum));

            SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, comparator, typeof(RandomDatum
                                                                                        ), typeof(RandomDatum), conf, metadata);
            sorter.Sort(new Path[] { unsortedFile }, sortedFile, false);
        }
예제 #5
0
 /// <summary>
 /// test
 /// <c>BloomMapFile.Writer</c>
 /// constructors
 /// </summary>
 public virtual void TestBloomMapFileConstructors()
 {
     BloomMapFile.Writer writer = null;
     try
     {
         FileSystem ts           = FileSystem.Get(conf);
         string     testFileName = TestFile.ToString();
         writer = new BloomMapFile.Writer(conf, ts, testFileName, typeof(IntWritable), typeof(
                                              Text), SequenceFile.CompressionType.Block, defaultCodec, defaultProgress);
         NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
                                          );
         writer.Close();
         writer = new BloomMapFile.Writer(conf, ts, testFileName, typeof(IntWritable), typeof(
                                              Text), SequenceFile.CompressionType.Block, defaultProgress);
         NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
                                          );
         writer.Close();
         writer = new BloomMapFile.Writer(conf, ts, testFileName, typeof(IntWritable), typeof(
                                              Text), SequenceFile.CompressionType.Block);
         NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
                                          );
         writer.Close();
         writer = new BloomMapFile.Writer(conf, ts, testFileName, typeof(IntWritable), typeof(
                                              Text), SequenceFile.CompressionType.Record, defaultCodec, defaultProgress);
         NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
                                          );
         writer.Close();
         writer = new BloomMapFile.Writer(conf, ts, testFileName, typeof(IntWritable), typeof(
                                              Text), SequenceFile.CompressionType.Record, defaultProgress);
         NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
                                          );
         writer.Close();
         writer = new BloomMapFile.Writer(conf, ts, testFileName, typeof(IntWritable), typeof(
                                              Text), SequenceFile.CompressionType.Record);
         NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
                                          );
         writer.Close();
         writer = new BloomMapFile.Writer(conf, ts, testFileName, WritableComparator.Get(typeof(
                                                                                             Text)), typeof(Text));
         NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
                                          );
         writer.Close();
     }
     catch (Exception)
     {
         Fail("testBloomMapFileConstructors error !!!");
     }
     finally
     {
         IOUtils.Cleanup(null, writer);
     }
 }
예제 #6
0
 /// <exception cref="System.IO.IOException"/>
 private static void WriteTest(FileSystem fs, RandomDatum[] data, string file, SequenceFile.CompressionType
                               compress)
 {
     MapFile.Delete(fs, file);
     Log.Info("creating with " + data.Length + " records");
     SetFile.Writer writer = new SetFile.Writer(conf, fs, file, WritableComparator.Get
                                                    (typeof(RandomDatum)), compress);
     for (int i = 0; i < data.Length; i++)
     {
         writer.Append(data[i]);
     }
     writer.Close();
 }
예제 #7
0
        /// <summary>Test that Writable's are configured by Comparator.</summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestConfigurableWritableComparator()
        {
            Configuration conf = new Configuration();

            conf.Set(TestWritableConfigParam, TestWritableConfigValue);
            WritableComparator wc = WritableComparator.Get(typeof(TestWritable.SimpleWritableComparable
                                                                  ), conf);

            TestWritable.SimpleWritableComparable key = ((TestWritable.SimpleWritableComparable
                                                          )wc.NewKey());
            NUnit.Framework.Assert.IsNotNull(wc.GetConf());
            NUnit.Framework.Assert.IsNotNull(key.GetConf());
            Assert.Equal(key.GetConf().Get(TestWritableConfigParam), TestWritableConfigValue
                         );
        }
예제 #8
0
        public virtual void TestDeprecatedConstructors()
        {
            string path = new Path(TestDir, "writes.mapfile").ToString();

            MapFile.Writer writer = null;
            MapFile.Reader reader = null;
            try
            {
                FileSystem fs = FileSystem.GetLocal(conf);
                writer = new MapFile.Writer(conf, fs, path, typeof(IntWritable), typeof(Text), SequenceFile.CompressionType
                                            .Record);
                NUnit.Framework.Assert.IsNotNull(writer);
                writer.Close();
                writer = new MapFile.Writer(conf, fs, path, typeof(IntWritable), typeof(Text), SequenceFile.CompressionType
                                            .Record, defaultProgressable);
                NUnit.Framework.Assert.IsNotNull(writer);
                writer.Close();
                writer = new MapFile.Writer(conf, fs, path, typeof(IntWritable), typeof(Text), SequenceFile.CompressionType
                                            .Record, defaultCodec, defaultProgressable);
                NUnit.Framework.Assert.IsNotNull(writer);
                writer.Close();
                writer = new MapFile.Writer(conf, fs, path, WritableComparator.Get(typeof(Text)),
                                            typeof(Text));
                NUnit.Framework.Assert.IsNotNull(writer);
                writer.Close();
                writer = new MapFile.Writer(conf, fs, path, WritableComparator.Get(typeof(Text)),
                                            typeof(Text), SequenceFile.CompressionType.Record);
                NUnit.Framework.Assert.IsNotNull(writer);
                writer.Close();
                writer = new MapFile.Writer(conf, fs, path, WritableComparator.Get(typeof(Text)),
                                            typeof(Text), SequenceFile.CompressionType.Record, defaultProgressable);
                NUnit.Framework.Assert.IsNotNull(writer);
                writer.Close();
                reader = new MapFile.Reader(fs, path, WritableComparator.Get(typeof(IntWritable))
                                            , conf);
                NUnit.Framework.Assert.IsNotNull(reader);
                NUnit.Framework.Assert.IsNotNull("reader key is null !!!", reader.GetKeyClass());
                NUnit.Framework.Assert.IsNotNull("reader value in null", reader.GetValueClass());
            }
            catch (IOException e)
            {
                NUnit.Framework.Assert.Fail(e.Message);
            }
            finally
            {
                IOUtils.Cleanup(null, writer, reader);
            }
        }
예제 #9
0
        /// <summary>
        /// Test a user comparator that relies on deserializing both arguments for each
        /// compare.
        /// </summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestShortWritableComparator()
        {
            ShortWritable writable1 = new ShortWritable((short)256);
            ShortWritable writable2 = new ShortWritable((short)128);
            ShortWritable writable3 = new ShortWritable((short)256);
            string        ShouldNotMatchWithResultOne = "Result should be 1, should not match the writables";

            Assert.True(ShouldNotMatchWithResultOne, writable1.CompareTo(writable2
                                                                         ) == 1);
            Assert.True(ShouldNotMatchWithResultOne, WritableComparator.Get
                            (typeof(ShortWritable)).Compare(writable1, writable2) == 1);
            string ShouldNotMatchWithResultMinusOne = "Result should be -1, should not match the writables";

            Assert.True(ShouldNotMatchWithResultMinusOne, writable2.CompareTo
                            (writable1) == -1);
            Assert.True(ShouldNotMatchWithResultMinusOne, WritableComparator
                        .Get(typeof(ShortWritable)).Compare(writable2, writable1) == -1);
            string ShouldMatch = "Result should be 0, should match the writables";

            Assert.True(ShouldMatch, writable1.CompareTo(writable1) == 0);
            Assert.True(ShouldMatch, WritableComparator.Get(typeof(ShortWritable
                                                                   )).Compare(writable1, writable3) == 0);
        }
예제 #10
0
 public Writer(Configuration conf, FileSystem fs, string dirName, WritableComparator
               comparator, Type valClass)
     : this(conf, new Path(dirName), Comparator(comparator), ValueClass(valClass))
 {
 }
예제 #11
0
 static MD5Hash()
 {
     // register this comparator
     WritableComparator.Define(typeof(MD5Hash), new MD5Hash.Comparator());
 }
예제 #12
0
 /// <summary>Compares this object with the specified object for order.</summary>
 public virtual int CompareTo(Org.Apache.Hadoop.IO.MD5Hash that)
 {
     return(WritableComparator.CompareBytes(this.digest, 0, Md5Len, that.digest, 0, Md5Len
                                            ));
 }
예제 #13
0
 static ShortWritable()
 {
     // register this comparator
     WritableComparator.Define(typeof(ShortWritable), new ShortWritable.Comparator());
 }
예제 #14
0
 static NullWritable()
 {
     // register this comparator
     WritableComparator.Define(typeof(NullWritable), new NullWritable.Comparator());
 }
예제 #15
0
 public virtual int CompareTo(Org.Apache.Hadoop.IO.RandomDatum o)
 {
     return(WritableComparator.CompareBytes(this.data, 0, this.length, o.data, 0, o.length
                                            ));
 }
예제 #16
0
 /// <exception cref="System.IO.IOException"/>
 private SetFile.Reader CreateReader(FileSystem fs)
 {
     return(new SetFile.Reader(fs, File, WritableComparator.Get(typeof(IntWritable)),
                               conf));
 }
예제 #17
0
 static Frob()
 {
     // register default comparator
     WritableComparator.Define(typeof(TestWritable.Frob), new TestWritable.FrobComparator
                                   ());
 }