public virtual void TestReaderGetClosest() { string TestMethodKey = "testReaderWithWrongKeyClass.mapfile"; MapFile.Writer writer = null; MapFile.Reader reader = null; try { writer = CreateWriter(TestMethodKey, typeof(IntWritable), typeof(Text)); for (int i = 0; i < 10; i++) { writer.Append(new IntWritable(i), new Text("value" + i)); } writer.Close(); reader = CreateReader(TestMethodKey, typeof(Text)); reader.GetClosest(new Text("2"), new Text(string.Empty)); NUnit.Framework.Assert.Fail("no excepted exception in testReaderWithWrongKeyClass !!!" ); } catch (IOException) { } finally { /* Should be thrown to pass the test */ IOUtils.Cleanup(null, writer, reader); } }
public virtual void TestOnFinalKey() { string TestMethodKey = "testOnFinalKey.mapfile"; int Size = 10; MapFile.Writer writer = null; MapFile.Reader reader = null; try { writer = CreateWriter(TestMethodKey, typeof(IntWritable), typeof(IntWritable)); for (int i = 0; i < Size; i++) { writer.Append(new IntWritable(i), new IntWritable(i)); } writer.Close(); reader = CreateReader(TestMethodKey, typeof(IntWritable)); IntWritable expectedKey = new IntWritable(0); reader.FinalKey(expectedKey); Assert.Equal("testOnFinalKey not same !!!", expectedKey, new IntWritable (9)); } catch (IOException) { NUnit.Framework.Assert.Fail("testOnFinalKey error !!!"); } finally { IOUtils.Cleanup(null, writer, reader); } }
public virtual void TestMidKeyOnCurrentApi() { // Write a mapfile of simple data: keys are string TestPrefix = "testMidKeyOnCurrentApi.mapfile"; MapFile.Writer writer = null; MapFile.Reader reader = null; try { writer = CreateWriter(TestPrefix, typeof(IntWritable), typeof(IntWritable)); // 0,1,....9 int Size = 10; for (int i = 0; i < Size; i++) { writer.Append(new IntWritable(i), new IntWritable(i)); } writer.Close(); reader = CreateReader(TestPrefix, typeof(IntWritable)); Assert.Equal(new IntWritable((Size - 1) / 2), reader.MidKey()); } finally { IOUtils.Cleanup(null, writer, reader); } }
public virtual void TestMerge() { string TestMethodKey = "testMerge.mapfile"; int Size = 10; int Iterations = 5; Path[] @in = new Path[5]; IList <int> expected = new AList <int>(); for (int j = 0; j < 5; j++) { using (MapFile.Writer writer = CreateWriter(TestMethodKey + "." + j, typeof(IntWritable ), typeof(Text))) { @in[j] = new Path(TestDir, TestMethodKey + "." + j); for (int i = 0; i < Size; i++) { expected.AddItem(i + j); writer.Append(new IntWritable(i + j), new Text("Value:" + (i + j))); } } } // Sort expected values expected.Sort(); // Merge all 5 files MapFile.Merger merger = new MapFile.Merger(conf); merger.Merge(@in, true, new Path(TestDir, TestMethodKey)); using (MapFile.Reader reader = CreateReader(TestMethodKey, typeof(IntWritable))) { int start = 0; // test iteration Text startValue = new Text("Value:" + start); int i = 0; while (i++ < Iterations) { IEnumerator <int> expectedIterator = expected.GetEnumerator(); IntWritable key = new IntWritable(start); Text value = startValue; IntWritable prev = new IntWritable(start); while (reader.Next(key, value)) { Assert.True("Next key should be always equal or more", prev.Get () <= key.Get()); Assert.Equal(expectedIterator.Next(), key.Get()); prev.Set(key.Get()); } reader.Reset(); } } // inputs should be deleted for (int j_1 = 0; j_1 < @in.Length; j_1++) { Path path = @in[j_1]; NUnit.Framework.Assert.IsFalse("inputs should be deleted", path.GetFileSystem(conf ).Exists(path)); } }
public virtual void TestGetClosestOnCurrentApi() { string TestPrefix = "testGetClosestOnCurrentApi.mapfile"; MapFile.Writer writer = null; MapFile.Reader reader = null; try { writer = CreateWriter(TestPrefix, typeof(Text), typeof(Text)); int FirstKey = 1; // Test keys: 11,21,31,...,91 for (int i = FirstKey; i < 100; i += 10) { Text t = new Text(Extensions.ToString(i)); writer.Append(t, t); } writer.Close(); reader = CreateReader(TestPrefix, typeof(Text)); Text key = new Text("55"); Text value = new Text(); // Test get closest with step forward Text closest = (Text)reader.GetClosest(key, value); Assert.Equal(new Text("61"), closest); // Test get closest with step back closest = (Text)reader.GetClosest(key, value, true); Assert.Equal(new Text("51"), closest); // Test get closest when we pass explicit key Text explicitKey = new Text("21"); closest = (Text)reader.GetClosest(explicitKey, value); Assert.Equal(new Text("21"), explicitKey); // Test what happens at boundaries. Assert if searching a key that is // less than first key in the mapfile, that the first key is returned. key = new Text("00"); closest = (Text)reader.GetClosest(key, value); Assert.Equal(FirstKey, System.Convert.ToInt32(closest.ToString ())); // Assert that null is returned if key is > last entry in mapfile. key = new Text("92"); closest = (Text)reader.GetClosest(key, value); NUnit.Framework.Assert.IsNull("Not null key in testGetClosestWithNewCode", closest ); // If we were looking for the key before, we should get the last key closest = (Text)reader.GetClosest(key, value, true); Assert.Equal(new Text("91"), closest); } finally { IOUtils.Cleanup(null, writer, reader); } }
/// <summary>Open the output generated by this format.</summary> /// <exception cref="System.IO.IOException"/> public static MapFile.Reader[] GetReaders(Path dir, Configuration conf) { FileSystem fs = dir.GetFileSystem(conf); Path[] names = FileUtil.Stat2Paths(fs.ListStatus(dir)); // sort names, so that hash partitioning works Arrays.Sort(names); MapFile.Reader[] parts = new MapFile.Reader[names.Length]; for (int i = 0; i < names.Length; i++) { parts[i] = new MapFile.Reader(fs, names[i].ToString(), conf); } return(parts); }
public virtual void TestDeprecatedConstructors() { string path = new Path(TestDir, "writes.mapfile").ToString(); MapFile.Writer writer = null; MapFile.Reader reader = null; try { FileSystem fs = FileSystem.GetLocal(conf); writer = new MapFile.Writer(conf, fs, path, typeof(IntWritable), typeof(Text), SequenceFile.CompressionType .Record); NUnit.Framework.Assert.IsNotNull(writer); writer.Close(); writer = new MapFile.Writer(conf, fs, path, typeof(IntWritable), typeof(Text), SequenceFile.CompressionType .Record, defaultProgressable); NUnit.Framework.Assert.IsNotNull(writer); writer.Close(); writer = new MapFile.Writer(conf, fs, path, typeof(IntWritable), typeof(Text), SequenceFile.CompressionType .Record, defaultCodec, defaultProgressable); NUnit.Framework.Assert.IsNotNull(writer); writer.Close(); writer = new MapFile.Writer(conf, fs, path, WritableComparator.Get(typeof(Text)), typeof(Text)); NUnit.Framework.Assert.IsNotNull(writer); writer.Close(); writer = new MapFile.Writer(conf, fs, path, WritableComparator.Get(typeof(Text)), typeof(Text), SequenceFile.CompressionType.Record); NUnit.Framework.Assert.IsNotNull(writer); writer.Close(); writer = new MapFile.Writer(conf, fs, path, WritableComparator.Get(typeof(Text)), typeof(Text), SequenceFile.CompressionType.Record, defaultProgressable); NUnit.Framework.Assert.IsNotNull(writer); writer.Close(); reader = new MapFile.Reader(fs, path, WritableComparator.Get(typeof(IntWritable)) , conf); NUnit.Framework.Assert.IsNotNull(reader); NUnit.Framework.Assert.IsNotNull("reader key is null !!!", reader.GetKeyClass()); NUnit.Framework.Assert.IsNotNull("reader value in null", reader.GetValueClass()); } catch (IOException e) { NUnit.Framework.Assert.Fail(e.Message); } finally { IOUtils.Cleanup(null, writer, reader); } }
public virtual void TestReaderKeyIteration() { string TestMethodKey = "testReaderKeyIteration.mapfile"; int Size = 10; int Iterations = 5; MapFile.Writer writer = null; MapFile.Reader reader = null; try { writer = CreateWriter(TestMethodKey, typeof(IntWritable), typeof(Text)); int start = 0; for (int i = 0; i < Size; i++) { writer.Append(new IntWritable(i), new Text("Value:" + i)); } writer.Close(); reader = CreateReader(TestMethodKey, typeof(IntWritable)); // test iteration Writable startValue = new Text("Value:" + start); int i_1 = 0; while (i_1++ < Iterations) { IntWritable key = new IntWritable(start); Writable value = startValue; while (reader.Next(key, value)) { NUnit.Framework.Assert.IsNotNull(key); NUnit.Framework.Assert.IsNotNull(value); } reader.Reset(); } Assert.True("reader seek error !!!", reader.Seek(new IntWritable (Size / 2))); NUnit.Framework.Assert.IsFalse("reader seek error !!!", reader.Seek(new IntWritable (Size * 2))); } catch (IOException) { NUnit.Framework.Assert.Fail("reader seek error !!!"); } finally { IOUtils.Cleanup(null, writer, reader); } }
public virtual void TestMidKeyEmpty() { // Write a mapfile of simple data: keys are Path dirName = new Path(TestDir, "testMidKeyEmpty.mapfile"); FileSystem fs = FileSystem.GetLocal(conf); Path qualifiedDirName = fs.MakeQualified(dirName); MapFile.Writer writer = new MapFile.Writer(conf, fs, qualifiedDirName.ToString(), typeof(IntWritable), typeof(IntWritable)); writer.Close(); // Now do getClosest on created mapfile. MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf); try { Assert.Equal(null, reader.MidKey()); } finally { reader.Close(); } }
/// <exception cref="System.Exception"/> private void CodecTestMapFile(Type clazz, SequenceFile.CompressionType type, int records) { FileSystem fs = FileSystem.Get(conf); Log.Info("Creating MapFiles with " + records + " records using codec " + clazz.Name ); Path path = new Path(new Path(Runtime.GetProperty("test.build.data", "/tmp")), clazz .Name + "-" + type + "-" + records); Log.Info("Writing " + path); CreateMapFile(conf, fs, path, System.Activator.CreateInstance(clazz), type, records ); MapFile.Reader reader = new MapFile.Reader(path, conf); Text key1 = new Text("002"); NUnit.Framework.Assert.IsNotNull(reader.Get(key1, new Text())); Text key2 = new Text("004"); NUnit.Framework.Assert.IsNotNull(reader.Get(key2, new Text())); }
public virtual void TestGetClosest() { // Write a mapfile of simple data: keys are Path dirName = new Path(TestDir, "testGetClosest.mapfile"); FileSystem fs = FileSystem.GetLocal(conf); Path qualifiedDirName = fs.MakeQualified(dirName); // Make an index entry for every third insertion. MapFile.Writer.SetIndexInterval(conf, 3); MapFile.Writer writer = null; MapFile.Reader reader = null; try { writer = new MapFile.Writer(conf, fs, qualifiedDirName.ToString(), typeof(Text), typeof(Text)); // Assert that the index interval is 1 Assert.Equal(3, writer.GetIndexInterval()); // Add entries up to 100 in intervals of ten. int FirstKey = 10; for (int i = FirstKey; i < 100; i += 10) { string iStr = Extensions.ToString(i); Text t = new Text(Runtime.Substring("00", iStr.Length) + iStr); writer.Append(t, t); } writer.Close(); // Now do getClosest on created mapfile. reader = new MapFile.Reader(qualifiedDirName, conf); Text key = new Text("55"); Text value = new Text(); Text closest = (Text)reader.GetClosest(key, value); // Assert that closest after 55 is 60 Assert.Equal(new Text("60"), closest); // Get closest that falls before the passed key: 50 closest = (Text)reader.GetClosest(key, value, true); Assert.Equal(new Text("50"), closest); // Test get closest when we pass explicit key Text Twenty = new Text("20"); closest = (Text)reader.GetClosest(Twenty, value); Assert.Equal(Twenty, closest); closest = (Text)reader.GetClosest(Twenty, value, true); Assert.Equal(Twenty, closest); // Test what happens at boundaries. Assert if searching a key that is // less than first key in the mapfile, that the first key is returned. key = new Text("00"); closest = (Text)reader.GetClosest(key, value); Assert.Equal(FirstKey, System.Convert.ToInt32(closest.ToString ())); // If we're looking for the first key before, and we pass in a key before // the first key in the file, we should get null closest = (Text)reader.GetClosest(key, value, true); NUnit.Framework.Assert.IsNull(closest); // Assert that null is returned if key is > last entry in mapfile. key = new Text("99"); closest = (Text)reader.GetClosest(key, value); NUnit.Framework.Assert.IsNull(closest); // If we were looking for the key before, we should get the last key closest = (Text)reader.GetClosest(key, value, true); Assert.Equal(new Text("90"), closest); } finally { IOUtils.Cleanup(null, writer, reader); } }