/// <summary>test none codecs</summary> /// <exception cref="System.IO.IOException"/> internal virtual void BasicWithSomeCodec(string codec) { Path ncTFile = new Path(Root, "basic.tfile"); FSDataOutputStream fout = CreateFSOutput(ncTFile); TFile.Writer writer = new TFile.Writer(fout, minBlockSize, codec, "memcmp", conf); WriteRecords(writer); fout.Close(); FSDataInputStream fin = fs.Open(ncTFile); TFile.Reader reader = new TFile.Reader(fs.Open(ncTFile), fs.GetFileStatus(ncTFile ).GetLen(), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); ReadAllRecords(scanner); scanner.SeekTo(GetSomeKey(50)); Assert.True("location lookup failed", scanner.SeekTo(GetSomeKey (50))); // read the key and see if it matches byte[] readKey = ReadKey(scanner); Assert.True("seeked key does not match", Arrays.Equals(GetSomeKey (50), readKey)); scanner.SeekTo(new byte[0]); byte[] val1 = ReadValue(scanner); scanner.SeekTo(new byte[0]); byte[] val2 = ReadValue(scanner); Assert.True(Arrays.Equals(val1, val2)); // check for lowerBound scanner.LowerBound(GetSomeKey(50)); Assert.True("locaton lookup failed", scanner.currentLocation.CompareTo (reader.End()) < 0); readKey = ReadKey(scanner); Assert.True("seeked key does not match", Arrays.Equals(readKey, GetSomeKey(50))); // check for upper bound scanner.UpperBound(GetSomeKey(50)); Assert.True("location lookup failed", scanner.currentLocation.CompareTo (reader.End()) < 0); readKey = ReadKey(scanner); Assert.True("seeked key does not match", Arrays.Equals(readKey, GetSomeKey(51))); scanner.Close(); // test for a range of scanner scanner = reader.CreateScannerByKey(GetSomeKey(10), GetSomeKey(60)); ReadAndCheckbytes(scanner, 10, 50); NUnit.Framework.Assert.IsFalse(scanner.Advance()); scanner.Close(); reader.Close(); fin.Close(); fs.Delete(ncTFile, true); }
/// <exception cref="System.IO.IOException"/> private void ReadValueBeforeKey(int recordIndex) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString (ComposeSortedKey(Key, recordIndex)), null); try { byte[] vbuf = new byte[BufSize]; int vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), Value + recordIndex); byte[] kbuf = new byte[BufSize]; int klen = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf); Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen), ComposeSortedKey(Key, recordIndex)); } finally { scanner.Close(); reader.Close(); } }
// no-op /// <exception cref="System.IO.IOException"/> public virtual void TestFailureNegativeOffset() { if (skip) { return; } WriteRecords(2, true, true); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); byte[] buf = new byte[K]; try { scanner.Entry().GetKey(buf, -1); NUnit.Framework.Assert.Fail("Failed to handle key negative offset."); } catch (Exception) { } finally { } // noop, expecting exceptions scanner.Close(); reader.Close(); }
/// <exception cref="System.IO.IOException"/> private void ReadKeyWithoutValue(int recordIndex) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString (ComposeSortedKey(Key, recordIndex)), null); try { // read the indexed key byte[] kbuf1 = new byte[BufSize]; int klen1 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf1); Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1 ), ComposeSortedKey(Key, recordIndex)); if (scanner.Advance() && !scanner.AtEnd()) { // read the next key following the indexed byte[] kbuf2 = new byte[BufSize]; int klen2 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf2); Assert.Equal(Runtime.GetStringForBytes(kbuf2, 0, klen2 ), ComposeSortedKey(Key, recordIndex + 1)); } } finally { scanner.Close(); reader.Close(); } }
/// <exception cref="System.IO.IOException"/> internal static void ReadRecords(FileSystem fs, Path path, int count, Configuration conf) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); try { for (int nx = 0; nx < count; nx++, scanner.Advance()) { NUnit.Framework.Assert.IsFalse(scanner.AtEnd()); // Assert.assertTrue(scanner.next()); byte[] kbuf = new byte[BufSize]; int klen = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf); Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen), ComposeSortedKey(Key, nx)); byte[] vbuf = new byte[BufSize]; int vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), Value + nx); } Assert.True(scanner.AtEnd()); NUnit.Framework.Assert.IsFalse(scanner.Advance()); } finally { scanner.Close(); reader.Close(); } }
/// <exception cref="System.IO.IOException"/> internal virtual void ReadFile() { long fileLength = fs.GetFileStatus(path).GetLen(); int numSplit = 10; long splitSize = fileLength / numSplit + 1; TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); long offset = 0; long rowCount = 0; BytesWritable key; BytesWritable value; for (int i = 0; i < numSplit; ++i, offset += splitSize) { TFile.Reader.Scanner scanner = reader.CreateScannerByByteRange(offset, splitSize); int count = 0; key = new BytesWritable(); value = new BytesWritable(); while (!scanner.AtEnd()) { scanner.Entry().Get(key, value); ++count; scanner.Advance(); } scanner.Close(); Assert.True(count > 0); rowCount += count; } Assert.Equal(rowCount, reader.GetEntryCount()); reader.Close(); }
public virtual void TestFailureNegativeLength_2() { if (skip) { return; } CloseOutput(); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); try { scanner.LowerBound(Runtime.GetBytesForString("keyX"), 0, -1); NUnit.Framework.Assert.Fail("Error on handling negative length."); } catch (Exception) { } finally { // noop, expecting exceptions scanner.Close(); reader.Close(); } CloseOutput(); }
public virtual void TestFailureReadValueManyTimes() { if (skip) { return; } WriteRecords(5); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); byte[] vbuf = new byte[BufSize]; int vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), Value + 0); try { scanner.Entry().GetValue(vbuf); NUnit.Framework.Assert.Fail("Cannot get the value mlutiple times."); } catch (Exception) { } // noop, expecting exceptions scanner.Close(); reader.Close(); }
/// <exception cref="System.IO.IOException"/> public virtual void TestFailureSeek() { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); try { // can't find ceil try { scanner.LowerBound(Runtime.GetBytesForString("keyN")); NUnit.Framework.Assert.Fail("Cannot search in a unsorted TFile!"); } catch (Exception) { } finally { } // noop, expecting excetions // can't find higher try { scanner.UpperBound(Runtime.GetBytesForString("keyA")); NUnit.Framework.Assert.Fail("Cannot search higher in a unsorted TFile!"); } catch (Exception) { } finally { } // noop, expecting excetions // can't seek try { scanner.SeekTo(Runtime.GetBytesForString("keyM")); NUnit.Framework.Assert.Fail("Cannot search a unsorted TFile!"); } catch (Exception) { } finally { } } finally { // noop, expecting excetions scanner.Close(); reader.Close(); } }
/// <exception cref="System.IO.IOException"/> private void CheckBlockIndex(int recordIndex, int blockIndexExpected) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); scanner.SeekTo(Runtime.GetBytesForString(ComposeSortedKey(Key, recordIndex ))); Assert.Equal(blockIndexExpected, scanner.currentLocation.GetBlockIndex ()); scanner.Close(); reader.Close(); }
public virtual void TestNoDataEntry() { if (skip) { return; } CloseOutput(); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); Assert.True(reader.IsSorted()); TFile.Reader.Scanner scanner = reader.CreateScanner(); Assert.True(scanner.AtEnd()); scanner.Close(); reader.Close(); }
// unsorted with some codec /// <exception cref="System.IO.IOException"/> internal virtual void UnsortedWithSomeCodec(string codec) { Path uTfile = new Path(Root, "unsorted.tfile"); FSDataOutputStream fout = CreateFSOutput(uTfile); TFile.Writer writer = new TFile.Writer(fout, minBlockSize, codec, null, conf); WriteRecords(writer); writer.Close(); fout.Close(); FSDataInputStream fin = fs.Open(uTfile); TFile.Reader reader = new TFile.Reader(fs.Open(uTfile), fs.GetFileStatus(uTfile). GetLen(), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); ReadAllRecords(scanner); scanner.Close(); reader.Close(); fin.Close(); fs.Delete(uTfile, true); }
public virtual void TestLocate() { if (skip) { return; } WriteRecords(3 * records1stBlock); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, 2))); Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, records1stBlock - 1))); Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, records1stBlock ))); TFile.Reader.Location locX = Locate(scanner, Runtime.GetBytesForString("keyX" )); Assert.Equal(scanner.endLocation, locX); scanner.Close(); reader.Close(); }
// we still can scan records in an unsorted TFile /// <exception cref="System.IO.IOException"/> public virtual void TestScan() { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); NUnit.Framework.Assert.IsFalse(reader.IsSorted()); Assert.Equal((int)reader.GetEntryCount(), 4); TFile.Reader.Scanner scanner = reader.CreateScanner(); try { // read key and value byte[] kbuf = new byte[BufSize]; int klen = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf); Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen), "keyZ"); byte[] vbuf = new byte[BufSize]; int vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), "valueZ"); scanner.Advance(); // now try get value first vbuf = new byte[BufSize]; vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), "valueM"); kbuf = new byte[BufSize]; klen = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf); Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen), "keyM"); } finally { scanner.Close(); reader.Close(); } }
public virtual void TestFailureNegativeLength_3() { if (skip) { return; } WriteRecords(3); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); try { // test negative array offset try { scanner.SeekTo(Runtime.GetBytesForString("keyY"), -1, 4); NUnit.Framework.Assert.Fail("Failed to handle negative offset."); } catch (Exception) { } // noop, expecting exceptions // test negative array length try { scanner.SeekTo(Runtime.GetBytesForString("keyY"), 0, -2); NUnit.Framework.Assert.Fail("Failed to handle negative key length."); } catch (Exception) { } } finally { // noop, expecting exceptions reader.Close(); scanner.Close(); } }
/* Similar to readFile(), tests the scanner created * by record numbers rather than the offsets. */ /// <exception cref="System.IO.IOException"/> internal virtual void ReadRowSplits(int numSplits) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); long totalRecords = reader.GetEntryCount(); for (int i = 0; i < numSplits; i++) { long startRec = i * totalRecords / numSplits; long endRec = (i + 1) * totalRecords / numSplits; if (i == numSplits - 1) { endRec = totalRecords; } TFile.Reader.Scanner scanner = reader.CreateScannerByRecordNum(startRec, endRec); int count = 0; BytesWritable key = new BytesWritable(); BytesWritable value = new BytesWritable(); long x = startRec; while (!scanner.AtEnd()) { Assert.Equal("Incorrect RecNum returned by scanner", scanner.GetRecordNum (), x); scanner.Entry().Get(key, value); ++count; Assert.Equal("Incorrect RecNum returned by scanner", scanner.GetRecordNum (), x); scanner.Advance(); ++x; } scanner.Close(); Assert.True(count == (endRec - startRec)); } // make sure specifying range at the end gives zero records. TFile.Reader.Scanner scanner_1 = reader.CreateScannerByRecordNum(totalRecords, -1 ); Assert.True(scanner_1.AtEnd()); }
/// <exception cref="System.IO.IOException"/> private void ReadValueWithoutKey(int recordIndex) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString (ComposeSortedKey(Key, recordIndex)), null); byte[] vbuf1 = new byte[BufSize]; int vlen1 = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf1); Assert.Equal(Runtime.GetStringForBytes(vbuf1, 0, vlen1 ), Value + recordIndex); if (scanner.Advance() && !scanner.AtEnd()) { byte[] vbuf2 = new byte[BufSize]; int vlen2 = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf2); Assert.Equal(Runtime.GetStringForBytes(vbuf2, 0, vlen2 ), Value + (recordIndex + 1)); } scanner.Close(); reader.Close(); }
/// <exception cref="System.IO.IOException"/> private void ReadKeyManyTimes(int recordIndex) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString (ComposeSortedKey(Key, recordIndex)), null); // read the indexed key byte[] kbuf1 = new byte[BufSize]; int klen1 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf1); Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1 ), ComposeSortedKey(Key, recordIndex)); klen1 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf1); Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1 ), ComposeSortedKey(Key, recordIndex)); klen1 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf1); Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1 ), ComposeSortedKey(Key, recordIndex)); scanner.Close(); reader.Close(); }
/// <exception cref="System.IO.IOException"/> public virtual void Close() { scanner.Close(); reader.Close(); fsdis.Close(); }