Exemplo n.º 1
0
        public virtual void TestReaderGetClosest()
        {
            string TestMethodKey = "testReaderWithWrongKeyClass.mapfile";

            MapFile.Writer writer = null;
            MapFile.Reader reader = null;
            try
            {
                writer = CreateWriter(TestMethodKey, typeof(IntWritable), typeof(Text));
                for (int i = 0; i < 10; i++)
                {
                    writer.Append(new IntWritable(i), new Text("value" + i));
                }
                writer.Close();
                reader = CreateReader(TestMethodKey, typeof(Text));
                reader.GetClosest(new Text("2"), new Text(string.Empty));
                NUnit.Framework.Assert.Fail("no excepted exception in testReaderWithWrongKeyClass !!!"
                                            );
            }
            catch (IOException)
            {
            }
            finally
            {
                /* Should be thrown to pass the test */
                IOUtils.Cleanup(null, writer, reader);
            }
        }
Exemplo n.º 2
0
        public virtual void TestOnFinalKey()
        {
            string TestMethodKey = "testOnFinalKey.mapfile";
            int    Size          = 10;

            MapFile.Writer writer = null;
            MapFile.Reader reader = null;
            try
            {
                writer = CreateWriter(TestMethodKey, typeof(IntWritable), typeof(IntWritable));
                for (int i = 0; i < Size; i++)
                {
                    writer.Append(new IntWritable(i), new IntWritable(i));
                }
                writer.Close();
                reader = CreateReader(TestMethodKey, typeof(IntWritable));
                IntWritable expectedKey = new IntWritable(0);
                reader.FinalKey(expectedKey);
                Assert.Equal("testOnFinalKey not same !!!", expectedKey, new IntWritable
                                 (9));
            }
            catch (IOException)
            {
                NUnit.Framework.Assert.Fail("testOnFinalKey error !!!");
            }
            finally
            {
                IOUtils.Cleanup(null, writer, reader);
            }
        }
Exemplo n.º 3
0
        public virtual void TestMidKeyOnCurrentApi()
        {
            // Write a mapfile of simple data: keys are
            string TestPrefix = "testMidKeyOnCurrentApi.mapfile";

            MapFile.Writer writer = null;
            MapFile.Reader reader = null;
            try
            {
                writer = CreateWriter(TestPrefix, typeof(IntWritable), typeof(IntWritable));
                // 0,1,....9
                int Size = 10;
                for (int i = 0; i < Size; i++)
                {
                    writer.Append(new IntWritable(i), new IntWritable(i));
                }
                writer.Close();
                reader = CreateReader(TestPrefix, typeof(IntWritable));
                Assert.Equal(new IntWritable((Size - 1) / 2), reader.MidKey());
            }
            finally
            {
                IOUtils.Cleanup(null, writer, reader);
            }
        }
Exemplo n.º 4
0
        public virtual void TestMerge()
        {
            string TestMethodKey = "testMerge.mapfile";
            int    Size          = 10;
            int    Iterations    = 5;

            Path[]      @in      = new Path[5];
            IList <int> expected = new AList <int>();

            for (int j = 0; j < 5; j++)
            {
                using (MapFile.Writer writer = CreateWriter(TestMethodKey + "." + j, typeof(IntWritable
                                                                                            ), typeof(Text)))
                {
                    @in[j] = new Path(TestDir, TestMethodKey + "." + j);
                    for (int i = 0; i < Size; i++)
                    {
                        expected.AddItem(i + j);
                        writer.Append(new IntWritable(i + j), new Text("Value:" + (i + j)));
                    }
                }
            }
            // Sort expected values
            expected.Sort();
            // Merge all 5 files
            MapFile.Merger merger = new MapFile.Merger(conf);
            merger.Merge(@in, true, new Path(TestDir, TestMethodKey));
            using (MapFile.Reader reader = CreateReader(TestMethodKey, typeof(IntWritable)))
            {
                int start = 0;
                // test iteration
                Text startValue = new Text("Value:" + start);
                int  i          = 0;
                while (i++ < Iterations)
                {
                    IEnumerator <int> expectedIterator = expected.GetEnumerator();
                    IntWritable       key = new IntWritable(start);
                    Text        value     = startValue;
                    IntWritable prev      = new IntWritable(start);
                    while (reader.Next(key, value))
                    {
                        Assert.True("Next key should be always equal or more", prev.Get
                                        () <= key.Get());
                        Assert.Equal(expectedIterator.Next(), key.Get());
                        prev.Set(key.Get());
                    }
                    reader.Reset();
                }
            }
            // inputs should be deleted
            for (int j_1 = 0; j_1 < @in.Length; j_1++)
            {
                Path path = @in[j_1];
                NUnit.Framework.Assert.IsFalse("inputs should be deleted", path.GetFileSystem(conf
                                                                                              ).Exists(path));
            }
        }
Exemplo n.º 5
0
        public virtual void TestGetClosestOnCurrentApi()
        {
            string TestPrefix = "testGetClosestOnCurrentApi.mapfile";

            MapFile.Writer writer = null;
            MapFile.Reader reader = null;
            try
            {
                writer = CreateWriter(TestPrefix, typeof(Text), typeof(Text));
                int FirstKey = 1;
                // Test keys: 11,21,31,...,91
                for (int i = FirstKey; i < 100; i += 10)
                {
                    Text t = new Text(Extensions.ToString(i));
                    writer.Append(t, t);
                }
                writer.Close();
                reader = CreateReader(TestPrefix, typeof(Text));
                Text key   = new Text("55");
                Text value = new Text();
                // Test get closest with step forward
                Text closest = (Text)reader.GetClosest(key, value);
                Assert.Equal(new Text("61"), closest);
                // Test get closest with step back
                closest = (Text)reader.GetClosest(key, value, true);
                Assert.Equal(new Text("51"), closest);
                // Test get closest when we pass explicit key
                Text explicitKey = new Text("21");
                closest = (Text)reader.GetClosest(explicitKey, value);
                Assert.Equal(new Text("21"), explicitKey);
                // Test what happens at boundaries. Assert if searching a key that is
                // less than first key in the mapfile, that the first key is returned.
                key     = new Text("00");
                closest = (Text)reader.GetClosest(key, value);
                Assert.Equal(FirstKey, System.Convert.ToInt32(closest.ToString
                                                                  ()));
                // Assert that null is returned if key is > last entry in mapfile.
                key     = new Text("92");
                closest = (Text)reader.GetClosest(key, value);
                NUnit.Framework.Assert.IsNull("Not null key in testGetClosestWithNewCode", closest
                                              );
                // If we were looking for the key before, we should get the last key
                closest = (Text)reader.GetClosest(key, value, true);
                Assert.Equal(new Text("91"), closest);
            }
            finally
            {
                IOUtils.Cleanup(null, writer, reader);
            }
        }
Exemplo n.º 6
0
        /// <summary>Open the output generated by this format.</summary>
        /// <exception cref="System.IO.IOException"/>
        public static MapFile.Reader[] GetReaders(Path dir, Configuration conf)
        {
            FileSystem fs = dir.GetFileSystem(conf);

            Path[] names = FileUtil.Stat2Paths(fs.ListStatus(dir));
            // sort names, so that hash partitioning works
            Arrays.Sort(names);
            MapFile.Reader[] parts = new MapFile.Reader[names.Length];
            for (int i = 0; i < names.Length; i++)
            {
                parts[i] = new MapFile.Reader(fs, names[i].ToString(), conf);
            }
            return(parts);
        }
Exemplo n.º 7
0
        public virtual void TestDeprecatedConstructors()
        {
            string path = new Path(TestDir, "writes.mapfile").ToString();

            MapFile.Writer writer = null;
            MapFile.Reader reader = null;
            try
            {
                FileSystem fs = FileSystem.GetLocal(conf);
                writer = new MapFile.Writer(conf, fs, path, typeof(IntWritable), typeof(Text), SequenceFile.CompressionType
                                            .Record);
                NUnit.Framework.Assert.IsNotNull(writer);
                writer.Close();
                writer = new MapFile.Writer(conf, fs, path, typeof(IntWritable), typeof(Text), SequenceFile.CompressionType
                                            .Record, defaultProgressable);
                NUnit.Framework.Assert.IsNotNull(writer);
                writer.Close();
                writer = new MapFile.Writer(conf, fs, path, typeof(IntWritable), typeof(Text), SequenceFile.CompressionType
                                            .Record, defaultCodec, defaultProgressable);
                NUnit.Framework.Assert.IsNotNull(writer);
                writer.Close();
                writer = new MapFile.Writer(conf, fs, path, WritableComparator.Get(typeof(Text)),
                                            typeof(Text));
                NUnit.Framework.Assert.IsNotNull(writer);
                writer.Close();
                writer = new MapFile.Writer(conf, fs, path, WritableComparator.Get(typeof(Text)),
                                            typeof(Text), SequenceFile.CompressionType.Record);
                NUnit.Framework.Assert.IsNotNull(writer);
                writer.Close();
                writer = new MapFile.Writer(conf, fs, path, WritableComparator.Get(typeof(Text)),
                                            typeof(Text), SequenceFile.CompressionType.Record, defaultProgressable);
                NUnit.Framework.Assert.IsNotNull(writer);
                writer.Close();
                reader = new MapFile.Reader(fs, path, WritableComparator.Get(typeof(IntWritable))
                                            , conf);
                NUnit.Framework.Assert.IsNotNull(reader);
                NUnit.Framework.Assert.IsNotNull("reader key is null !!!", reader.GetKeyClass());
                NUnit.Framework.Assert.IsNotNull("reader value in null", reader.GetValueClass());
            }
            catch (IOException e)
            {
                NUnit.Framework.Assert.Fail(e.Message);
            }
            finally
            {
                IOUtils.Cleanup(null, writer, reader);
            }
        }
Exemplo n.º 8
0
        public virtual void TestReaderKeyIteration()
        {
            string TestMethodKey = "testReaderKeyIteration.mapfile";
            int    Size          = 10;
            int    Iterations    = 5;

            MapFile.Writer writer = null;
            MapFile.Reader reader = null;
            try
            {
                writer = CreateWriter(TestMethodKey, typeof(IntWritable), typeof(Text));
                int start = 0;
                for (int i = 0; i < Size; i++)
                {
                    writer.Append(new IntWritable(i), new Text("Value:" + i));
                }
                writer.Close();
                reader = CreateReader(TestMethodKey, typeof(IntWritable));
                // test iteration
                Writable startValue = new Text("Value:" + start);
                int      i_1        = 0;
                while (i_1++ < Iterations)
                {
                    IntWritable key   = new IntWritable(start);
                    Writable    value = startValue;
                    while (reader.Next(key, value))
                    {
                        NUnit.Framework.Assert.IsNotNull(key);
                        NUnit.Framework.Assert.IsNotNull(value);
                    }
                    reader.Reset();
                }
                Assert.True("reader seek error !!!", reader.Seek(new IntWritable
                                                                     (Size / 2)));
                NUnit.Framework.Assert.IsFalse("reader seek error !!!", reader.Seek(new IntWritable
                                                                                        (Size * 2)));
            }
            catch (IOException)
            {
                NUnit.Framework.Assert.Fail("reader seek error !!!");
            }
            finally
            {
                IOUtils.Cleanup(null, writer, reader);
            }
        }
Exemplo n.º 9
0
        public virtual void TestMidKeyEmpty()
        {
            // Write a mapfile of simple data: keys are
            Path       dirName          = new Path(TestDir, "testMidKeyEmpty.mapfile");
            FileSystem fs               = FileSystem.GetLocal(conf);
            Path       qualifiedDirName = fs.MakeQualified(dirName);

            MapFile.Writer writer = new MapFile.Writer(conf, fs, qualifiedDirName.ToString(),
                                                       typeof(IntWritable), typeof(IntWritable));
            writer.Close();
            // Now do getClosest on created mapfile.
            MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf);
            try
            {
                Assert.Equal(null, reader.MidKey());
            }
            finally
            {
                reader.Close();
            }
        }
Exemplo n.º 10
0
        /// <exception cref="System.Exception"/>
        private void CodecTestMapFile(Type clazz, SequenceFile.CompressionType type, int
                                      records)
        {
            FileSystem fs = FileSystem.Get(conf);

            Log.Info("Creating MapFiles with " + records + " records using codec " + clazz.Name
                     );
            Path path = new Path(new Path(Runtime.GetProperty("test.build.data", "/tmp")), clazz
                                 .Name + "-" + type + "-" + records);

            Log.Info("Writing " + path);
            CreateMapFile(conf, fs, path, System.Activator.CreateInstance(clazz), type, records
                          );
            MapFile.Reader reader = new MapFile.Reader(path, conf);
            Text           key1   = new Text("002");

            NUnit.Framework.Assert.IsNotNull(reader.Get(key1, new Text()));
            Text key2 = new Text("004");

            NUnit.Framework.Assert.IsNotNull(reader.Get(key2, new Text()));
        }
Exemplo n.º 11
0
        public virtual void TestGetClosest()
        {
            // Write a mapfile of simple data: keys are
            Path       dirName          = new Path(TestDir, "testGetClosest.mapfile");
            FileSystem fs               = FileSystem.GetLocal(conf);
            Path       qualifiedDirName = fs.MakeQualified(dirName);

            // Make an index entry for every third insertion.
            MapFile.Writer.SetIndexInterval(conf, 3);
            MapFile.Writer writer = null;
            MapFile.Reader reader = null;
            try
            {
                writer = new MapFile.Writer(conf, fs, qualifiedDirName.ToString(), typeof(Text),
                                            typeof(Text));
                // Assert that the index interval is 1
                Assert.Equal(3, writer.GetIndexInterval());
                // Add entries up to 100 in intervals of ten.
                int FirstKey = 10;
                for (int i = FirstKey; i < 100; i += 10)
                {
                    string iStr = Extensions.ToString(i);
                    Text   t    = new Text(Runtime.Substring("00", iStr.Length) + iStr);
                    writer.Append(t, t);
                }
                writer.Close();
                // Now do getClosest on created mapfile.
                reader = new MapFile.Reader(qualifiedDirName, conf);
                Text key     = new Text("55");
                Text value   = new Text();
                Text closest = (Text)reader.GetClosest(key, value);
                // Assert that closest after 55 is 60
                Assert.Equal(new Text("60"), closest);
                // Get closest that falls before the passed key: 50
                closest = (Text)reader.GetClosest(key, value, true);
                Assert.Equal(new Text("50"), closest);
                // Test get closest when we pass explicit key
                Text Twenty = new Text("20");
                closest = (Text)reader.GetClosest(Twenty, value);
                Assert.Equal(Twenty, closest);
                closest = (Text)reader.GetClosest(Twenty, value, true);
                Assert.Equal(Twenty, closest);
                // Test what happens at boundaries. Assert if searching a key that is
                // less than first key in the mapfile, that the first key is returned.
                key     = new Text("00");
                closest = (Text)reader.GetClosest(key, value);
                Assert.Equal(FirstKey, System.Convert.ToInt32(closest.ToString
                                                                  ()));
                // If we're looking for the first key before, and we pass in a key before
                // the first key in the file, we should get null
                closest = (Text)reader.GetClosest(key, value, true);
                NUnit.Framework.Assert.IsNull(closest);
                // Assert that null is returned if key is > last entry in mapfile.
                key     = new Text("99");
                closest = (Text)reader.GetClosest(key, value);
                NUnit.Framework.Assert.IsNull(closest);
                // If we were looking for the key before, we should get the last key
                closest = (Text)reader.GetClosest(key, value, true);
                Assert.Equal(new Text("90"), closest);
            }
            finally
            {
                IOUtils.Cleanup(null, writer, reader);
            }
        }