public virtual void TestFailureNegativeLength_2() { if (skip) { return; } CloseOutput(); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); try { scanner.LowerBound(Runtime.GetBytesForString("keyX"), 0, -1); NUnit.Framework.Assert.Fail("Error on handling negative length."); } catch (Exception) { } finally { // noop, expecting exceptions scanner.Close(); reader.Close(); } CloseOutput(); }
public virtual void TestFailureGetNonExistentMetaBlock() { if (skip) { return; } writer.Append(Runtime.GetBytesForString("keyX"), Runtime.GetBytesForString ("valueX")); // create a new metablock DataOutputStream outMeta = writer.PrepareMetaBlock("testX", Compression.Algorithm .Gz.GetName()); outMeta.Write(123); outMeta.Write(Runtime.GetBytesForString("foo")); outMeta.Close(); CloseOutput(); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); DataInputStream mb = reader.GetMetaBlock("testX"); NUnit.Framework.Assert.IsNotNull(mb); mb.Close(); try { DataInputStream mbBad = reader.GetMetaBlock("testY"); NUnit.Framework.Assert.Fail("Error on handling non-existent metablocks."); } catch (Exception) { } // noop, expecting exceptions reader.Close(); }
/// <exception cref="System.IO.IOException"/> private void ReadValueBeforeKey(int recordIndex) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString (ComposeSortedKey(Key, recordIndex)), null); try { byte[] vbuf = new byte[BufSize]; int vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), Value + recordIndex); byte[] kbuf = new byte[BufSize]; int klen = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf); Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen), ComposeSortedKey(Key, recordIndex)); } finally { scanner.Close(); reader.Close(); } }
/// <exception cref="System.IO.IOException"/> internal static void ReadRecords(FileSystem fs, Path path, int count, Configuration conf) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); try { for (int nx = 0; nx < count; nx++, scanner.Advance()) { NUnit.Framework.Assert.IsFalse(scanner.AtEnd()); // Assert.assertTrue(scanner.next()); byte[] kbuf = new byte[BufSize]; int klen = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf); Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen), ComposeSortedKey(Key, nx)); byte[] vbuf = new byte[BufSize]; int vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), Value + nx); } Assert.True(scanner.AtEnd()); NUnit.Framework.Assert.IsFalse(scanner.Advance()); } finally { scanner.Close(); reader.Close(); } }
/// <exception cref="System.IO.IOException"/> internal virtual void ReadFile() { long fileLength = fs.GetFileStatus(path).GetLen(); int numSplit = 10; long splitSize = fileLength / numSplit + 1; TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); long offset = 0; long rowCount = 0; BytesWritable key; BytesWritable value; for (int i = 0; i < numSplit; ++i, offset += splitSize) { TFile.Reader.Scanner scanner = reader.CreateScannerByByteRange(offset, splitSize); int count = 0; key = new BytesWritable(); value = new BytesWritable(); while (!scanner.AtEnd()) { scanner.Entry().Get(key, value); ++count; scanner.Advance(); } scanner.Close(); Assert.True(count > 0); rowCount += count; } Assert.Equal(rowCount, reader.GetEntryCount()); reader.Close(); }
// no-op /// <exception cref="System.IO.IOException"/> public virtual void TestFailureNegativeOffset() { if (skip) { return; } WriteRecords(2, true, true); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); byte[] buf = new byte[K]; try { scanner.Entry().GetKey(buf, -1); NUnit.Framework.Assert.Fail("Failed to handle key negative offset."); } catch (Exception) { } finally { } // noop, expecting exceptions scanner.Close(); reader.Close(); }
public virtual void TestFailureReadValueManyTimes() { if (skip) { return; } WriteRecords(5); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); byte[] vbuf = new byte[BufSize]; int vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), Value + 0); try { scanner.Entry().GetValue(vbuf); NUnit.Framework.Assert.Fail("Cannot get the value mlutiple times."); } catch (Exception) { } // noop, expecting exceptions scanner.Close(); reader.Close(); }
/// <exception cref="System.IO.IOException"/> private void ReadKeyWithoutValue(int recordIndex) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString (ComposeSortedKey(Key, recordIndex)), null); try { // read the indexed key byte[] kbuf1 = new byte[BufSize]; int klen1 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf1); Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1 ), ComposeSortedKey(Key, recordIndex)); if (scanner.Advance() && !scanner.AtEnd()) { // read the next key following the indexed byte[] kbuf2 = new byte[BufSize]; int klen2 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf2); Assert.Equal(Runtime.GetStringForBytes(kbuf2, 0, klen2 ), ComposeSortedKey(Key, recordIndex + 1)); } } finally { scanner.Close(); reader.Close(); } }
/// <exception cref="System.IO.IOException"/> public LogReader(Configuration conf, Path remoteAppLogFile) { FileContext fileContext = FileContext.GetFileContext(conf); this.fsDataIStream = fileContext.Open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, fileContext.GetFileStatus(remoteAppLogFile ).GetLen(), conf); this.scanner = reader.CreateScanner(); }
/// <exception cref="System.IO.IOException"/> public TFileReadable(FileSystem fs, Path path, int osBufferSize, Configuration conf ) { this.fsdis = fs.Open(path, osBufferSize); this.reader = new TFile.Reader(fsdis, fs.GetFileStatus(path).GetLen(), conf); this.scanner = reader.CreateScanner(); keyBuffer = new byte[32]; valueBuffer = new byte[32]; }
/// <exception cref="System.IO.IOException"/> public HistoryFileReader(FileSystemApplicationHistoryStore _enclosing, Path historyFile ) { this._enclosing = _enclosing; this.fsdis = this._enclosing.fs.Open(historyFile); this.reader = new TFile.Reader(this.fsdis, this._enclosing.fs.GetFileStatus(historyFile ).GetLen(), this._enclosing.GetConfig()); this.Reset(); }
/// <exception cref="System.IO.IOException"/> internal virtual void CheckRecNums() { long fileLen = fs.GetFileStatus(path).GetLen(); TFile.Reader reader = new TFile.Reader(fs.Open(path), fileLen, conf); long totalRecs = reader.GetEntryCount(); long begin = random.NextLong() % (totalRecs / 2); if (begin < 0) { begin += (totalRecs / 2); } long end = random.NextLong() % (totalRecs / 2); if (end < 0) { end += (totalRecs / 2); } end += (totalRecs / 2) + 1; Assert.Equal("RecNum for offset=0 should be 0", 0, reader.GetRecordNumNear (0)); foreach (long x in new long[] { fileLen, fileLen + 1, 2 * fileLen }) { Assert.Equal("RecNum for offset>=fileLen should be total entries" , totalRecs, reader.GetRecordNumNear(x)); } for (long i = 0; i < 100; ++i) { Assert.Equal("Locaton to RecNum conversion not symmetric", i, reader.GetRecordNumByLocation(reader.GetLocationByRecordNum(i))); } for (long i_1 = 1; i_1 < 100; ++i_1) { long x_1 = totalRecs - i_1; Assert.Equal("Locaton to RecNum conversion not symmetric", x_1 , reader.GetRecordNumByLocation(reader.GetLocationByRecordNum(x_1))); } for (long i_2 = begin; i_2 < end; ++i_2) { Assert.Equal("Locaton to RecNum conversion not symmetric", i_2 , reader.GetRecordNumByLocation(reader.GetLocationByRecordNum(i_2))); } for (int i_3 = 0; i_3 < 1000; ++i_3) { long x_1 = random.NextLong() % totalRecs; if (x_1 < 0) { x_1 += totalRecs; } Assert.Equal("Locaton to RecNum conversion not symmetric", x_1 , reader.GetRecordNumByLocation(reader.GetLocationByRecordNum(x_1))); } }
/// <exception cref="System.IO.IOException"/> private void CheckBlockIndex(int recordIndex, int blockIndexExpected) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); scanner.SeekTo(Runtime.GetBytesForString(ComposeSortedKey(Key, recordIndex ))); Assert.Equal(blockIndexExpected, scanner.currentLocation.GetBlockIndex ()); scanner.Close(); reader.Close(); }
/// <exception cref="System.IO.IOException"/> public virtual void TestFailureSeek() { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); try { // can't find ceil try { scanner.LowerBound(Runtime.GetBytesForString("keyN")); NUnit.Framework.Assert.Fail("Cannot search in a unsorted TFile!"); } catch (Exception) { } finally { } // noop, expecting excetions // can't find higher try { scanner.UpperBound(Runtime.GetBytesForString("keyA")); NUnit.Framework.Assert.Fail("Cannot search higher in a unsorted TFile!"); } catch (Exception) { } finally { } // noop, expecting excetions // can't seek try { scanner.SeekTo(Runtime.GetBytesForString("keyM")); NUnit.Framework.Assert.Fail("Cannot search a unsorted TFile!"); } catch (Exception) { } finally { } } finally { // noop, expecting excetions scanner.Close(); reader.Close(); } }
/// <exception cref="System.IO.IOException"/> private void ReadNumMetablocks(TFile.Reader reader, int n) { int len = Runtime.GetBytesForString(("something to test" + 0)).Length; for (int i = 0; i < n; i++) { DataInputStream din = reader.GetMetaBlock("TfileMeta" + i); byte[] b = new byte[len]; din.ReadFully(b); Assert.True("faield to match metadata", Arrays.Equals(Runtime.GetBytesForString (("something to test" + i)), b)); din.Close(); } }
/// <summary>test none codecs</summary> /// <exception cref="System.IO.IOException"/> internal virtual void BasicWithSomeCodec(string codec) { Path ncTFile = new Path(Root, "basic.tfile"); FSDataOutputStream fout = CreateFSOutput(ncTFile); TFile.Writer writer = new TFile.Writer(fout, minBlockSize, codec, "memcmp", conf); WriteRecords(writer); fout.Close(); FSDataInputStream fin = fs.Open(ncTFile); TFile.Reader reader = new TFile.Reader(fs.Open(ncTFile), fs.GetFileStatus(ncTFile ).GetLen(), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); ReadAllRecords(scanner); scanner.SeekTo(GetSomeKey(50)); Assert.True("location lookup failed", scanner.SeekTo(GetSomeKey (50))); // read the key and see if it matches byte[] readKey = ReadKey(scanner); Assert.True("seeked key does not match", Arrays.Equals(GetSomeKey (50), readKey)); scanner.SeekTo(new byte[0]); byte[] val1 = ReadValue(scanner); scanner.SeekTo(new byte[0]); byte[] val2 = ReadValue(scanner); Assert.True(Arrays.Equals(val1, val2)); // check for lowerBound scanner.LowerBound(GetSomeKey(50)); Assert.True("locaton lookup failed", scanner.currentLocation.CompareTo (reader.End()) < 0); readKey = ReadKey(scanner); Assert.True("seeked key does not match", Arrays.Equals(readKey, GetSomeKey(50))); // check for upper bound scanner.UpperBound(GetSomeKey(50)); Assert.True("location lookup failed", scanner.currentLocation.CompareTo (reader.End()) < 0); readKey = ReadKey(scanner); Assert.True("seeked key does not match", Arrays.Equals(readKey, GetSomeKey(51))); scanner.Close(); // test for a range of scanner scanner = reader.CreateScannerByKey(GetSomeKey(10), GetSomeKey(60)); ReadAndCheckbytes(scanner, 10, 50); NUnit.Framework.Assert.IsFalse(scanner.Advance()); scanner.Close(); reader.Close(); fin.Close(); fs.Delete(ncTFile, true); }
public virtual void TestNoDataEntry() { if (skip) { return; } CloseOutput(); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); Assert.True(reader.IsSorted()); TFile.Reader.Scanner scanner = reader.CreateScanner(); Assert.True(scanner.AtEnd()); scanner.Close(); reader.Close(); }
// test meta blocks for tfiles /// <exception cref="System.IO.IOException"/> public virtual void TestMetaBlocks() { Path mFile = new Path(Root, "meta.tfile"); FSDataOutputStream fout = CreateFSOutput(mFile); TFile.Writer writer = new TFile.Writer(fout, minBlockSize, "none", null, conf); SomeTestingWithMetaBlock(writer, "none"); writer.Close(); fout.Close(); FSDataInputStream fin = fs.Open(mFile); TFile.Reader reader = new TFile.Reader(fin, fs.GetFileStatus(mFile).GetLen(), conf ); SomeReadingWithMetaBlock(reader); fs.Delete(mFile, true); reader.Close(); fin.Close(); }
/// <exception cref="System.IO.IOException"/> private void SomeReadingWithMetaBlock(TFile.Reader reader) { DataInputStream din = null; ReadNumMetablocks(reader, 10); try { din = reader.GetMetaBlock("NO ONE"); Assert.True(false); } catch (MetaBlockDoesNotExist) { } // should catch din = reader.GetMetaBlock("TFileMeta100"); int read = din.Read(); Assert.True("check for status", (read == -1)); din.Close(); }
// unsorted with some codec /// <exception cref="System.IO.IOException"/> internal virtual void UnsortedWithSomeCodec(string codec) { Path uTfile = new Path(Root, "unsorted.tfile"); FSDataOutputStream fout = CreateFSOutput(uTfile); TFile.Writer writer = new TFile.Writer(fout, minBlockSize, codec, null, conf); WriteRecords(writer); writer.Close(); fout.Close(); FSDataInputStream fin = fs.Open(uTfile); TFile.Reader reader = new TFile.Reader(fs.Open(uTfile), fs.GetFileStatus(uTfile). GetLen(), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); ReadAllRecords(scanner); scanner.Close(); reader.Close(); fin.Close(); fs.Delete(uTfile, true); }
/// <exception cref="System.IO.IOException"/> public virtual void SeekTFile() { int miss = 0; long totalBytes = 0; FSDataInputStream fsdis = fs.Open(path); TFile.Reader reader = new TFile.Reader(fsdis, fs.GetFileStatus(path).GetLen(), conf ); KeySampler kSampler = new KeySampler(rng, reader.GetFirstKey(), reader.GetLastKey (), keyLenGen); TFile.Reader.Scanner scanner = reader.CreateScanner(); BytesWritable key = new BytesWritable(); BytesWritable val = new BytesWritable(); timer.Reset(); timer.Start(); for (int i = 0; i < options.seekCount; ++i) { kSampler.Next(key); scanner.LowerBound(key.Get(), 0, key.GetSize()); if (!scanner.AtEnd()) { scanner.Entry().Get(key, val); totalBytes += key.GetSize(); totalBytes += val.GetSize(); } else { ++miss; } } timer.Stop(); double duration = (double)timer.Read() / 1000; // in us. System.Console.Out.Printf("time: %s...avg seek: %s...%d hit...%d miss...avg I/O size: %.2fKB\n" , timer.ToString(), NanoTimer.NanoTimeToString(timer.Read() / options.seekCount) , options.seekCount - miss, miss, (double)totalBytes / 1024 / (options.seekCount - miss)); }
// we still can scan records in an unsorted TFile /// <exception cref="System.IO.IOException"/> public virtual void TestFailureScannerWithKeys() { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); NUnit.Framework.Assert.IsFalse(reader.IsSorted()); Assert.Equal((int)reader.GetEntryCount(), 4); try { TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString ("aaa"), Runtime.GetBytesForString("zzz")); NUnit.Framework.Assert.Fail("Failed to catch creating scanner with keys on unsorted file." ); } catch (RuntimeException) { } finally { reader.Close(); } }
public virtual void TestLocate() { if (skip) { return; } WriteRecords(3 * records1stBlock); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, 2))); Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, records1stBlock - 1))); Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, records1stBlock ))); TFile.Reader.Location locX = Locate(scanner, Runtime.GetBytesForString("keyX" )); Assert.Equal(scanner.endLocation, locX); scanner.Close(); reader.Close(); }
// we still can scan records in an unsorted TFile /// <exception cref="System.IO.IOException"/> public virtual void TestScan() { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); NUnit.Framework.Assert.IsFalse(reader.IsSorted()); Assert.Equal((int)reader.GetEntryCount(), 4); TFile.Reader.Scanner scanner = reader.CreateScanner(); try { // read key and value byte[] kbuf = new byte[BufSize]; int klen = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf); Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen), "keyZ"); byte[] vbuf = new byte[BufSize]; int vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), "valueZ"); scanner.Advance(); // now try get value first vbuf = new byte[BufSize]; vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), "valueM"); kbuf = new byte[BufSize]; klen = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf); Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen), "keyM"); } finally { scanner.Close(); reader.Close(); } }
public virtual void TestFailureNegativeLength_3() { if (skip) { return; } WriteRecords(3); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); try { // test negative array offset try { scanner.SeekTo(Runtime.GetBytesForString("keyY"), -1, 4); NUnit.Framework.Assert.Fail("Failed to handle negative offset."); } catch (Exception) { } // noop, expecting exceptions // test negative array length try { scanner.SeekTo(Runtime.GetBytesForString("keyY"), 0, -2); NUnit.Framework.Assert.Fail("Failed to handle negative key length."); } catch (Exception) { } } finally { // noop, expecting exceptions reader.Close(); scanner.Close(); } }
/* Similar to readFile(), tests the scanner created * by record numbers rather than the offsets. */ /// <exception cref="System.IO.IOException"/> internal virtual void ReadRowSplits(int numSplits) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); long totalRecords = reader.GetEntryCount(); for (int i = 0; i < numSplits; i++) { long startRec = i * totalRecords / numSplits; long endRec = (i + 1) * totalRecords / numSplits; if (i == numSplits - 1) { endRec = totalRecords; } TFile.Reader.Scanner scanner = reader.CreateScannerByRecordNum(startRec, endRec); int count = 0; BytesWritable key = new BytesWritable(); BytesWritable value = new BytesWritable(); long x = startRec; while (!scanner.AtEnd()) { Assert.Equal("Incorrect RecNum returned by scanner", scanner.GetRecordNum (), x); scanner.Entry().Get(key, value); ++count; Assert.Equal("Incorrect RecNum returned by scanner", scanner.GetRecordNum (), x); scanner.Advance(); ++x; } scanner.Close(); Assert.True(count == (endRec - startRec)); } // make sure specifying range at the end gives zero records. TFile.Reader.Scanner scanner_1 = reader.CreateScannerByRecordNum(totalRecords, -1 ); Assert.True(scanner_1.AtEnd()); }
/// <exception cref="System.IO.IOException"/> private void ReadValueWithoutKey(int recordIndex) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString (ComposeSortedKey(Key, recordIndex)), null); byte[] vbuf1 = new byte[BufSize]; int vlen1 = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf1); Assert.Equal(Runtime.GetStringForBytes(vbuf1, 0, vlen1 ), Value + recordIndex); if (scanner.Advance() && !scanner.AtEnd()) { byte[] vbuf2 = new byte[BufSize]; int vlen2 = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf2); Assert.Equal(Runtime.GetStringForBytes(vbuf2, 0, vlen2 ), Value + (recordIndex + 1)); } scanner.Close(); reader.Close(); }
public virtual void TestFailureWriterNotClosed() { if (skip) { return; } TFile.Reader reader = null; try { reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen(), conf); NUnit.Framework.Assert.Fail("Cannot read before closing the writer."); } catch (IOException) { } finally { // noop, expecting exceptions if (reader != null) { reader.Close(); } } }
/// <exception cref="System.IO.IOException"/> private void ReadKeyManyTimes(int recordIndex) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString (ComposeSortedKey(Key, recordIndex)), null); // read the indexed key byte[] kbuf1 = new byte[BufSize]; int klen1 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf1); Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1 ), ComposeSortedKey(Key, recordIndex)); klen1 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf1); Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1 ), ComposeSortedKey(Key, recordIndex)); klen1 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf1); Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1 ), ComposeSortedKey(Key, recordIndex)); scanner.Close(); reader.Close(); }
/// <summary>Dump information about TFile.</summary> /// <param name="file">Path string of the TFile</param> /// <param name="out">PrintStream to output the information.</param> /// <param name="conf">The configuration object.</param> /// <exception cref="System.IO.IOException"/> public static void DumpInfo(string file, TextWriter @out, Configuration conf) { int maxKeySampleLen = 16; Path path = new Path(file); FileSystem fs = path.GetFileSystem(conf); long length = fs.GetFileStatus(path).GetLen(); FSDataInputStream fsdis = fs.Open(path); TFile.Reader reader = new TFile.Reader(fsdis, length, conf); try { LinkedHashMap <string, string> properties = new LinkedHashMap <string, string>(); int blockCnt = reader.readerBCF.GetBlockCount(); int metaBlkCnt = reader.readerBCF.metaIndex.index.Count; properties["BCFile Version"] = reader.readerBCF.version.ToString(); properties["TFile Version"] = reader.tfileMeta.version.ToString(); properties["File Length"] = System.Convert.ToString(length); properties["Data Compression"] = reader.readerBCF.GetDefaultCompressionName(); properties["Record Count"] = System.Convert.ToString(reader.GetEntryCount()); properties["Sorted"] = bool.ToString(reader.IsSorted()); if (reader.IsSorted()) { properties["Comparator"] = reader.GetComparatorName(); } properties["Data Block Count"] = Extensions.ToString(blockCnt); long dataSize = 0; long dataSizeUncompressed = 0; if (blockCnt > 0) { for (int i = 0; i < blockCnt; ++i) { BCFile.BlockRegion region = reader.readerBCF.dataIndex.GetBlockRegionList()[i]; dataSize += region.GetCompressedSize(); dataSizeUncompressed += region.GetRawSize(); } properties["Data Block Bytes"] = System.Convert.ToString(dataSize); if (reader.readerBCF.GetDefaultCompressionName() != "none") { properties["Data Block Uncompressed Bytes"] = System.Convert.ToString(dataSizeUncompressed ); properties["Data Block Compression Ratio"] = string.Format("1:%.1f", (double)dataSizeUncompressed / dataSize); } } properties["Meta Block Count"] = Extensions.ToString(metaBlkCnt); long metaSize = 0; long metaSizeUncompressed = 0; if (metaBlkCnt > 0) { ICollection <BCFile.MetaIndexEntry> metaBlks = reader.readerBCF.metaIndex.index.Values; bool calculateCompression = false; for (IEnumerator <BCFile.MetaIndexEntry> it = metaBlks.GetEnumerator(); it.HasNext ();) { BCFile.MetaIndexEntry e = it.Next(); metaSize += e.GetRegion().GetCompressedSize(); metaSizeUncompressed += e.GetRegion().GetRawSize(); if (e.GetCompressionAlgorithm() != Compression.Algorithm.None) { calculateCompression = true; } } properties["Meta Block Bytes"] = System.Convert.ToString(metaSize); if (calculateCompression) { properties["Meta Block Uncompressed Bytes"] = System.Convert.ToString(metaSizeUncompressed ); properties["Meta Block Compression Ratio"] = string.Format("1:%.1f", (double)metaSizeUncompressed / metaSize); } } properties["Meta-Data Size Ratio"] = string.Format("1:%.1f", (double)dataSize / metaSize ); long leftOverBytes = length - dataSize - metaSize; long miscSize = BCFile.Magic.Size() * 2 + long.Size / byte.Size + Utils.Version.Size (); long metaIndexSize = leftOverBytes - miscSize; properties["Meta Block Index Bytes"] = System.Convert.ToString(metaIndexSize); properties["Headers Etc Bytes"] = System.Convert.ToString(miscSize); // Now output the properties table. int maxKeyLength = 0; ICollection <KeyValuePair <string, string> > entrySet = properties; for (IEnumerator <KeyValuePair <string, string> > it_1 = entrySet.GetEnumerator(); it_1 .HasNext();) { KeyValuePair <string, string> e = it_1.Next(); if (e.Key.Length > maxKeyLength) { maxKeyLength = e.Key.Length; } } for (IEnumerator <KeyValuePair <string, string> > it_2 = entrySet.GetEnumerator(); it_2 .HasNext();) { KeyValuePair <string, string> e = it_2.Next(); @out.Printf("%s : %s%n", TFileDumper.Align.Format(e.Key, maxKeyLength, TFileDumper.Align .Left), e.Value); } @out.WriteLine(); reader.CheckTFileDataIndex(); if (blockCnt > 0) { string blkID = "Data-Block"; int blkIDWidth = TFileDumper.Align.CalculateWidth(blkID, blockCnt); int blkIDWidth2 = TFileDumper.Align.CalculateWidth(string.Empty, blockCnt); string offset = "Offset"; int offsetWidth = TFileDumper.Align.CalculateWidth(offset, length); string blkLen = "Length"; int blkLenWidth = TFileDumper.Align.CalculateWidth(blkLen, dataSize / blockCnt * 10); string rawSize = "Raw-Size"; int rawSizeWidth = TFileDumper.Align.CalculateWidth(rawSize, dataSizeUncompressed / blockCnt * 10); string records = "Records"; int recordsWidth = TFileDumper.Align.CalculateWidth(records, reader.GetEntryCount () / blockCnt * 10); string endKey = "End-Key"; int endKeyWidth = Math.Max(endKey.Length, maxKeySampleLen * 2 + 5); @out.Printf("%s %s %s %s %s %s%n", TFileDumper.Align.Format(blkID, blkIDWidth, TFileDumper.Align .Center), TFileDumper.Align.Format(offset, offsetWidth, TFileDumper.Align.Center ), TFileDumper.Align.Format(blkLen, blkLenWidth, TFileDumper.Align.Center), TFileDumper.Align .Format(rawSize, rawSizeWidth, TFileDumper.Align.Center), TFileDumper.Align.Format (records, recordsWidth, TFileDumper.Align.Center), TFileDumper.Align.Format(endKey , endKeyWidth, TFileDumper.Align.Left)); for (int i = 0; i < blockCnt; ++i) { BCFile.BlockRegion region = reader.readerBCF.dataIndex.GetBlockRegionList()[i]; TFile.TFileIndexEntry indexEntry = reader.tfileIndex.GetEntry(i); @out.Printf("%s %s %s %s %s ", TFileDumper.Align.Format(TFileDumper.Align.Format( i, blkIDWidth2, TFileDumper.Align.ZeroPadded), blkIDWidth, TFileDumper.Align.Left ), TFileDumper.Align.Format(region.GetOffset(), offsetWidth, TFileDumper.Align.Left ), TFileDumper.Align.Format(region.GetCompressedSize(), blkLenWidth, TFileDumper.Align .Left), TFileDumper.Align.Format(region.GetRawSize(), rawSizeWidth, TFileDumper.Align .Left), TFileDumper.Align.Format(indexEntry.kvEntries, recordsWidth, TFileDumper.Align .Left)); byte[] key = indexEntry.key; bool asAscii = true; int sampleLen = Math.Min(maxKeySampleLen, key.Length); for (int j = 0; j < sampleLen; ++j) { byte b = key[j]; if ((((sbyte)b) < 32 && b != 9) || (b == 127)) { asAscii = false; } } if (!asAscii) { @out.Write("0X"); for (int j_1 = 0; j_1 < sampleLen; ++j_1) { byte b = key[i]; @out.Printf("%X", b); } } else { @out.Write(new string(key, 0, sampleLen, Charsets.Utf8)); } if (sampleLen < key.Length) { @out.Write("..."); } @out.WriteLine(); } } @out.WriteLine(); if (metaBlkCnt > 0) { string name = "Meta-Block"; int maxNameLen = 0; ICollection <KeyValuePair <string, BCFile.MetaIndexEntry> > metaBlkEntrySet = reader .readerBCF.metaIndex.index; for (IEnumerator <KeyValuePair <string, BCFile.MetaIndexEntry> > it = metaBlkEntrySet .GetEnumerator(); it_2.HasNext();) { KeyValuePair <string, BCFile.MetaIndexEntry> e = it_2.Next(); if (e.Key.Length > maxNameLen) { maxNameLen = e.Key.Length; } } int nameWidth = Math.Max(name.Length, maxNameLen); string offset = "Offset"; int offsetWidth = TFileDumper.Align.CalculateWidth(offset, length); string blkLen = "Length"; int blkLenWidth = TFileDumper.Align.CalculateWidth(blkLen, metaSize / metaBlkCnt * 10); string rawSize = "Raw-Size"; int rawSizeWidth = TFileDumper.Align.CalculateWidth(rawSize, metaSizeUncompressed / metaBlkCnt * 10); string compression = "Compression"; int compressionWidth = compression.Length; @out.Printf("%s %s %s %s %s%n", TFileDumper.Align.Format(name, nameWidth, TFileDumper.Align .Center), TFileDumper.Align.Format(offset, offsetWidth, TFileDumper.Align.Center ), TFileDumper.Align.Format(blkLen, blkLenWidth, TFileDumper.Align.Center), TFileDumper.Align .Format(rawSize, rawSizeWidth, TFileDumper.Align.Center), TFileDumper.Align.Format (compression, compressionWidth, TFileDumper.Align.Left)); for (IEnumerator <KeyValuePair <string, BCFile.MetaIndexEntry> > it_3 = metaBlkEntrySet .GetEnumerator(); it_3.HasNext();) { KeyValuePair <string, BCFile.MetaIndexEntry> e = it_3.Next(); string blkName = e.Value.GetMetaName(); BCFile.BlockRegion region = e.Value.GetRegion(); string blkCompression = e.Value.GetCompressionAlgorithm().GetName(); @out.Printf("%s %s %s %s %s%n", TFileDumper.Align.Format(blkName, nameWidth, TFileDumper.Align .Left), TFileDumper.Align.Format(region.GetOffset(), offsetWidth, TFileDumper.Align .Left), TFileDumper.Align.Format(region.GetCompressedSize(), blkLenWidth, TFileDumper.Align .Left), TFileDumper.Align.Format(region.GetRawSize(), rawSizeWidth, TFileDumper.Align .Left), TFileDumper.Align.Format(blkCompression, compressionWidth, TFileDumper.Align .Left)); } } } finally { IOUtils.Cleanup(Log, reader, fsdis); } }