public virtual void TestFailureNegativeLength_2()
{
if (skip)
{
return;
}
CloseOutput();
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
scanner.LowerBound(Runtime.GetBytesForString("keyX"), 0, -1);
NUnit.Framework.Assert.Fail("Error on handling negative length.");
}
catch (Exception)
{
}
finally
{
// noop, expecting exceptions
scanner.Close();
reader.Close();
}
CloseOutput();
}
public virtual void TestFailureGetNonExistentMetaBlock()
{
if (skip)
{
return;
}
writer.Append(Runtime.GetBytesForString("keyX"), Runtime.GetBytesForString
("valueX"));
// create a new metablock
DataOutputStream outMeta = writer.PrepareMetaBlock("testX", Compression.Algorithm
.Gz.GetName());
outMeta.Write(123);
outMeta.Write(Runtime.GetBytesForString("foo"));
outMeta.Close();
CloseOutput();
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
DataInputStream mb = reader.GetMetaBlock("testX");
NUnit.Framework.Assert.IsNotNull(mb);
mb.Close();
try
{
DataInputStream mbBad = reader.GetMetaBlock("testY");
NUnit.Framework.Assert.Fail("Error on handling non-existent metablocks.");
}
catch (Exception)
{
}
// noop, expecting exceptions
reader.Close();
}
/// <exception cref="System.IO.IOException"/>
private void ReadValueBeforeKey(int recordIndex)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString
(ComposeSortedKey(Key, recordIndex)), null);
try
{
byte[] vbuf = new byte[BufSize];
int vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
Value + recordIndex);
byte[] kbuf = new byte[BufSize];
int klen = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf);
Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen),
ComposeSortedKey(Key, recordIndex));
}
finally
{
scanner.Close();
reader.Close();
}
}
/// <exception cref="System.IO.IOException"/>
internal static void ReadRecords(FileSystem fs, Path path, int count, Configuration
conf)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
for (int nx = 0; nx < count; nx++, scanner.Advance())
{
NUnit.Framework.Assert.IsFalse(scanner.AtEnd());
// Assert.assertTrue(scanner.next());
byte[] kbuf = new byte[BufSize];
int klen = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf);
Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen),
ComposeSortedKey(Key, nx));
byte[] vbuf = new byte[BufSize];
int vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
Value + nx);
}
Assert.True(scanner.AtEnd());
NUnit.Framework.Assert.IsFalse(scanner.Advance());
}
finally
{
scanner.Close();
reader.Close();
}
}
/// <exception cref="System.IO.IOException"/>
internal virtual void ReadFile()
{
long fileLength = fs.GetFileStatus(path).GetLen();
int numSplit = 10;
long splitSize = fileLength / numSplit + 1;
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
long offset = 0;
long rowCount = 0;
BytesWritable key;
BytesWritable value;
for (int i = 0; i < numSplit; ++i, offset += splitSize)
{
TFile.Reader.Scanner scanner = reader.CreateScannerByByteRange(offset, splitSize);
int count = 0;
key = new BytesWritable();
value = new BytesWritable();
while (!scanner.AtEnd())
{
scanner.Entry().Get(key, value);
++count;
scanner.Advance();
}
scanner.Close();
Assert.True(count > 0);
rowCount += count;
}
Assert.Equal(rowCount, reader.GetEntryCount());
reader.Close();
}
// no-op
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailureNegativeOffset()
{
if (skip)
{
return;
}
WriteRecords(2, true, true);
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
byte[] buf = new byte[K];
try
{
scanner.Entry().GetKey(buf, -1);
NUnit.Framework.Assert.Fail("Failed to handle key negative offset.");
}
catch (Exception)
{
}
finally
{
}
// noop, expecting exceptions
scanner.Close();
reader.Close();
}
public virtual void TestFailureReadValueManyTimes()
{
if (skip)
{
return;
}
WriteRecords(5);
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
byte[] vbuf = new byte[BufSize];
int vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
Value + 0);
try
{
scanner.Entry().GetValue(vbuf);
NUnit.Framework.Assert.Fail("Cannot get the value mlutiple times.");
}
catch (Exception)
{
}
// noop, expecting exceptions
scanner.Close();
reader.Close();
}
/// <exception cref="System.IO.IOException"/>
private void ReadKeyWithoutValue(int recordIndex)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString
(ComposeSortedKey(Key, recordIndex)), null);
try
{
// read the indexed key
byte[] kbuf1 = new byte[BufSize];
int klen1 = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf1);
Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1
), ComposeSortedKey(Key, recordIndex));
if (scanner.Advance() && !scanner.AtEnd())
{
// read the next key following the indexed
byte[] kbuf2 = new byte[BufSize];
int klen2 = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf2);
Assert.Equal(Runtime.GetStringForBytes(kbuf2, 0, klen2
), ComposeSortedKey(Key, recordIndex + 1));
}
}
finally
{
scanner.Close();
reader.Close();
}
}
/// <exception cref="System.IO.IOException"/>
public LogReader(Configuration conf, Path remoteAppLogFile)
{
FileContext fileContext = FileContext.GetFileContext(conf);
this.fsDataIStream = fileContext.Open(remoteAppLogFile);
reader = new TFile.Reader(this.fsDataIStream, fileContext.GetFileStatus(remoteAppLogFile
).GetLen(), conf);
this.scanner = reader.CreateScanner();
}
/// <exception cref="System.IO.IOException"/>
public TFileReadable(FileSystem fs, Path path, int osBufferSize, Configuration conf
)
{
this.fsdis = fs.Open(path, osBufferSize);
this.reader = new TFile.Reader(fsdis, fs.GetFileStatus(path).GetLen(), conf);
this.scanner = reader.CreateScanner();
keyBuffer = new byte[32];
valueBuffer = new byte[32];
}
/// <exception cref="System.IO.IOException"/>
public HistoryFileReader(FileSystemApplicationHistoryStore _enclosing, Path historyFile
)
{
this._enclosing = _enclosing;
this.fsdis = this._enclosing.fs.Open(historyFile);
this.reader = new TFile.Reader(this.fsdis, this._enclosing.fs.GetFileStatus(historyFile
).GetLen(), this._enclosing.GetConfig());
this.Reset();
}
/// <exception cref="System.IO.IOException"/>
internal virtual void CheckRecNums()
{
long fileLen = fs.GetFileStatus(path).GetLen();
TFile.Reader reader = new TFile.Reader(fs.Open(path), fileLen, conf);
long totalRecs = reader.GetEntryCount();
long begin = random.NextLong() % (totalRecs / 2);
if (begin < 0)
{
begin += (totalRecs / 2);
}
long end = random.NextLong() % (totalRecs / 2);
if (end < 0)
{
end += (totalRecs / 2);
}
end += (totalRecs / 2) + 1;
Assert.Equal("RecNum for offset=0 should be 0", 0, reader.GetRecordNumNear
(0));
foreach (long x in new long[] { fileLen, fileLen + 1, 2 * fileLen })
{
Assert.Equal("RecNum for offset>=fileLen should be total entries"
, totalRecs, reader.GetRecordNumNear(x));
}
for (long i = 0; i < 100; ++i)
{
Assert.Equal("Locaton to RecNum conversion not symmetric", i,
reader.GetRecordNumByLocation(reader.GetLocationByRecordNum(i)));
}
for (long i_1 = 1; i_1 < 100; ++i_1)
{
long x_1 = totalRecs - i_1;
Assert.Equal("Locaton to RecNum conversion not symmetric", x_1
, reader.GetRecordNumByLocation(reader.GetLocationByRecordNum(x_1)));
}
for (long i_2 = begin; i_2 < end; ++i_2)
{
Assert.Equal("Locaton to RecNum conversion not symmetric", i_2
, reader.GetRecordNumByLocation(reader.GetLocationByRecordNum(i_2)));
}
for (int i_3 = 0; i_3 < 1000; ++i_3)
{
long x_1 = random.NextLong() % totalRecs;
if (x_1 < 0)
{
x_1 += totalRecs;
}
Assert.Equal("Locaton to RecNum conversion not symmetric", x_1
, reader.GetRecordNumByLocation(reader.GetLocationByRecordNum(x_1)));
}
}
/// <exception cref="System.IO.IOException"/>
private void CheckBlockIndex(int recordIndex, int blockIndexExpected)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
scanner.SeekTo(Runtime.GetBytesForString(ComposeSortedKey(Key, recordIndex
)));
Assert.Equal(blockIndexExpected, scanner.currentLocation.GetBlockIndex
());
scanner.Close();
reader.Close();
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailureSeek()
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
// can't find ceil
try
{
scanner.LowerBound(Runtime.GetBytesForString("keyN"));
NUnit.Framework.Assert.Fail("Cannot search in a unsorted TFile!");
}
catch (Exception)
{
}
finally
{
}
// noop, expecting excetions
// can't find higher
try
{
scanner.UpperBound(Runtime.GetBytesForString("keyA"));
NUnit.Framework.Assert.Fail("Cannot search higher in a unsorted TFile!");
}
catch (Exception)
{
}
finally
{
}
// noop, expecting excetions
// can't seek
try
{
scanner.SeekTo(Runtime.GetBytesForString("keyM"));
NUnit.Framework.Assert.Fail("Cannot search a unsorted TFile!");
}
catch (Exception)
{
}
finally
{
}
}
finally
{
// noop, expecting excetions
scanner.Close();
reader.Close();
}
}
/// <exception cref="System.IO.IOException"/>
private void ReadNumMetablocks(TFile.Reader reader, int n)
{
int len = Runtime.GetBytesForString(("something to test" + 0)).Length;
for (int i = 0; i < n; i++)
{
DataInputStream din = reader.GetMetaBlock("TfileMeta" + i);
byte[] b = new byte[len];
din.ReadFully(b);
Assert.True("faield to match metadata", Arrays.Equals(Runtime.GetBytesForString
(("something to test" + i)), b));
din.Close();
}
}
/// <summary>test none codecs</summary>
/// <exception cref="System.IO.IOException"/>
internal virtual void BasicWithSomeCodec(string codec)
{
Path ncTFile = new Path(Root, "basic.tfile");
FSDataOutputStream fout = CreateFSOutput(ncTFile);
TFile.Writer writer = new TFile.Writer(fout, minBlockSize, codec, "memcmp", conf);
WriteRecords(writer);
fout.Close();
FSDataInputStream fin = fs.Open(ncTFile);
TFile.Reader reader = new TFile.Reader(fs.Open(ncTFile), fs.GetFileStatus(ncTFile
).GetLen(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
ReadAllRecords(scanner);
scanner.SeekTo(GetSomeKey(50));
Assert.True("location lookup failed", scanner.SeekTo(GetSomeKey
(50)));
// read the key and see if it matches
byte[] readKey = ReadKey(scanner);
Assert.True("seeked key does not match", Arrays.Equals(GetSomeKey
(50), readKey));
scanner.SeekTo(new byte[0]);
byte[] val1 = ReadValue(scanner);
scanner.SeekTo(new byte[0]);
byte[] val2 = ReadValue(scanner);
Assert.True(Arrays.Equals(val1, val2));
// check for lowerBound
scanner.LowerBound(GetSomeKey(50));
Assert.True("locaton lookup failed", scanner.currentLocation.CompareTo
(reader.End()) < 0);
readKey = ReadKey(scanner);
Assert.True("seeked key does not match", Arrays.Equals(readKey,
GetSomeKey(50)));
// check for upper bound
scanner.UpperBound(GetSomeKey(50));
Assert.True("location lookup failed", scanner.currentLocation.CompareTo
(reader.End()) < 0);
readKey = ReadKey(scanner);
Assert.True("seeked key does not match", Arrays.Equals(readKey,
GetSomeKey(51)));
scanner.Close();
// test for a range of scanner
scanner = reader.CreateScannerByKey(GetSomeKey(10), GetSomeKey(60));
ReadAndCheckbytes(scanner, 10, 50);
NUnit.Framework.Assert.IsFalse(scanner.Advance());
scanner.Close();
reader.Close();
fin.Close();
fs.Delete(ncTFile, true);
}
public virtual void TestNoDataEntry()
{
if (skip)
{
return;
}
CloseOutput();
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
Assert.True(reader.IsSorted());
TFile.Reader.Scanner scanner = reader.CreateScanner();
Assert.True(scanner.AtEnd());
scanner.Close();
reader.Close();
}
// test meta blocks for tfiles
/// <exception cref="System.IO.IOException"/>
public virtual void TestMetaBlocks()
{
Path mFile = new Path(Root, "meta.tfile");
FSDataOutputStream fout = CreateFSOutput(mFile);
TFile.Writer writer = new TFile.Writer(fout, minBlockSize, "none", null, conf);
SomeTestingWithMetaBlock(writer, "none");
writer.Close();
fout.Close();
FSDataInputStream fin = fs.Open(mFile);
TFile.Reader reader = new TFile.Reader(fin, fs.GetFileStatus(mFile).GetLen(), conf
);
SomeReadingWithMetaBlock(reader);
fs.Delete(mFile, true);
reader.Close();
fin.Close();
}
/// <exception cref="System.IO.IOException"/>
private void SomeReadingWithMetaBlock(TFile.Reader reader)
{
DataInputStream din = null;
ReadNumMetablocks(reader, 10);
try
{
din = reader.GetMetaBlock("NO ONE");
Assert.True(false);
}
catch (MetaBlockDoesNotExist)
{
}
// should catch
din = reader.GetMetaBlock("TFileMeta100");
int read = din.Read();
Assert.True("check for status", (read == -1));
din.Close();
}
// unsorted with some codec
/// <exception cref="System.IO.IOException"/>
internal virtual void UnsortedWithSomeCodec(string codec)
{
Path uTfile = new Path(Root, "unsorted.tfile");
FSDataOutputStream fout = CreateFSOutput(uTfile);
TFile.Writer writer = new TFile.Writer(fout, minBlockSize, codec, null, conf);
WriteRecords(writer);
writer.Close();
fout.Close();
FSDataInputStream fin = fs.Open(uTfile);
TFile.Reader reader = new TFile.Reader(fs.Open(uTfile), fs.GetFileStatus(uTfile).
GetLen(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
ReadAllRecords(scanner);
scanner.Close();
reader.Close();
fin.Close();
fs.Delete(uTfile, true);
}
/// <exception cref="System.IO.IOException"/>
public virtual void SeekTFile()
{
int miss = 0;
long totalBytes = 0;
FSDataInputStream fsdis = fs.Open(path);
TFile.Reader reader = new TFile.Reader(fsdis, fs.GetFileStatus(path).GetLen(), conf
);
KeySampler kSampler = new KeySampler(rng, reader.GetFirstKey(), reader.GetLastKey
(), keyLenGen);
TFile.Reader.Scanner scanner = reader.CreateScanner();
BytesWritable key = new BytesWritable();
BytesWritable val = new BytesWritable();
timer.Reset();
timer.Start();
for (int i = 0; i < options.seekCount; ++i)
{
kSampler.Next(key);
scanner.LowerBound(key.Get(), 0, key.GetSize());
if (!scanner.AtEnd())
{
scanner.Entry().Get(key, val);
totalBytes += key.GetSize();
totalBytes += val.GetSize();
}
else
{
++miss;
}
}
timer.Stop();
double duration = (double)timer.Read() / 1000;
// in us.
System.Console.Out.Printf("time: %s...avg seek: %s...%d hit...%d miss...avg I/O size: %.2fKB\n"
, timer.ToString(), NanoTimer.NanoTimeToString(timer.Read() / options.seekCount)
, options.seekCount - miss, miss, (double)totalBytes / 1024 / (options.seekCount
- miss));
}
// we still can scan records in an unsorted TFile
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailureScannerWithKeys()
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
NUnit.Framework.Assert.IsFalse(reader.IsSorted());
Assert.Equal((int)reader.GetEntryCount(), 4);
try
{
TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString
("aaa"), Runtime.GetBytesForString("zzz"));
NUnit.Framework.Assert.Fail("Failed to catch creating scanner with keys on unsorted file."
);
}
catch (RuntimeException)
{
}
finally
{
reader.Close();
}
}
public virtual void TestLocate()
{
if (skip)
{
return;
}
WriteRecords(3 * records1stBlock);
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, 2)));
Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, records1stBlock
- 1)));
Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, records1stBlock
)));
TFile.Reader.Location locX = Locate(scanner, Runtime.GetBytesForString("keyX"
));
Assert.Equal(scanner.endLocation, locX);
scanner.Close();
reader.Close();
}
// we still can scan records in an unsorted TFile
/// <exception cref="System.IO.IOException"/>
public virtual void TestScan()
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
NUnit.Framework.Assert.IsFalse(reader.IsSorted());
Assert.Equal((int)reader.GetEntryCount(), 4);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
// read key and value
byte[] kbuf = new byte[BufSize];
int klen = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf);
Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen),
"keyZ");
byte[] vbuf = new byte[BufSize];
int vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
"valueZ");
scanner.Advance();
// now try get value first
vbuf = new byte[BufSize];
vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
"valueM");
kbuf = new byte[BufSize];
klen = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf);
Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen),
"keyM");
}
finally
{
scanner.Close();
reader.Close();
}
}
public virtual void TestFailureNegativeLength_3()
{
if (skip)
{
return;
}
WriteRecords(3);
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
// test negative array offset
try
{
scanner.SeekTo(Runtime.GetBytesForString("keyY"), -1, 4);
NUnit.Framework.Assert.Fail("Failed to handle negative offset.");
}
catch (Exception)
{
}
// noop, expecting exceptions
// test negative array length
try
{
scanner.SeekTo(Runtime.GetBytesForString("keyY"), 0, -2);
NUnit.Framework.Assert.Fail("Failed to handle negative key length.");
}
catch (Exception)
{
}
}
finally
{
// noop, expecting exceptions
reader.Close();
scanner.Close();
}
}
/* Similar to readFile(), tests the scanner created
* by record numbers rather than the offsets.
*/
/// <exception cref="System.IO.IOException"/>
internal virtual void ReadRowSplits(int numSplits)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
long totalRecords = reader.GetEntryCount();
for (int i = 0; i < numSplits; i++)
{
long startRec = i * totalRecords / numSplits;
long endRec = (i + 1) * totalRecords / numSplits;
if (i == numSplits - 1)
{
endRec = totalRecords;
}
TFile.Reader.Scanner scanner = reader.CreateScannerByRecordNum(startRec, endRec);
int count = 0;
BytesWritable key = new BytesWritable();
BytesWritable value = new BytesWritable();
long x = startRec;
while (!scanner.AtEnd())
{
Assert.Equal("Incorrect RecNum returned by scanner", scanner.GetRecordNum
(), x);
scanner.Entry().Get(key, value);
++count;
Assert.Equal("Incorrect RecNum returned by scanner", scanner.GetRecordNum
(), x);
scanner.Advance();
++x;
}
scanner.Close();
Assert.True(count == (endRec - startRec));
}
// make sure specifying range at the end gives zero records.
TFile.Reader.Scanner scanner_1 = reader.CreateScannerByRecordNum(totalRecords, -1
);
Assert.True(scanner_1.AtEnd());
}
/// <exception cref="System.IO.IOException"/>
private void ReadValueWithoutKey(int recordIndex)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString
(ComposeSortedKey(Key, recordIndex)), null);
byte[] vbuf1 = new byte[BufSize];
int vlen1 = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf1);
Assert.Equal(Runtime.GetStringForBytes(vbuf1, 0, vlen1
), Value + recordIndex);
if (scanner.Advance() && !scanner.AtEnd())
{
byte[] vbuf2 = new byte[BufSize];
int vlen2 = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf2);
Assert.Equal(Runtime.GetStringForBytes(vbuf2, 0, vlen2
), Value + (recordIndex + 1));
}
scanner.Close();
reader.Close();
}
public virtual void TestFailureWriterNotClosed()
{
if (skip)
{
return;
}
TFile.Reader reader = null;
try
{
reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen(), conf);
NUnit.Framework.Assert.Fail("Cannot read before closing the writer.");
}
catch (IOException)
{
}
finally
{
// noop, expecting exceptions
if (reader != null)
{
reader.Close();
}
}
}
/// <exception cref="System.IO.IOException"/>
private void ReadKeyManyTimes(int recordIndex)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString
(ComposeSortedKey(Key, recordIndex)), null);
// read the indexed key
byte[] kbuf1 = new byte[BufSize];
int klen1 = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf1);
Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1
), ComposeSortedKey(Key, recordIndex));
klen1 = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf1);
Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1
), ComposeSortedKey(Key, recordIndex));
klen1 = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf1);
Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1
), ComposeSortedKey(Key, recordIndex));
scanner.Close();
reader.Close();
}
/// <summary>Dump information about TFile.</summary>
/// <param name="file">Path string of the TFile</param>
/// <param name="out">PrintStream to output the information.</param>
/// <param name="conf">The configuration object.</param>
/// <exception cref="System.IO.IOException"/>
public static void DumpInfo(string file, TextWriter @out, Configuration conf)
{
int maxKeySampleLen = 16;
Path path = new Path(file);
FileSystem fs = path.GetFileSystem(conf);
long length = fs.GetFileStatus(path).GetLen();
FSDataInputStream fsdis = fs.Open(path);
TFile.Reader reader = new TFile.Reader(fsdis, length, conf);
try
{
LinkedHashMap <string, string> properties = new LinkedHashMap <string, string>();
int blockCnt = reader.readerBCF.GetBlockCount();
int metaBlkCnt = reader.readerBCF.metaIndex.index.Count;
properties["BCFile Version"] = reader.readerBCF.version.ToString();
properties["TFile Version"] = reader.tfileMeta.version.ToString();
properties["File Length"] = System.Convert.ToString(length);
properties["Data Compression"] = reader.readerBCF.GetDefaultCompressionName();
properties["Record Count"] = System.Convert.ToString(reader.GetEntryCount());
properties["Sorted"] = bool.ToString(reader.IsSorted());
if (reader.IsSorted())
{
properties["Comparator"] = reader.GetComparatorName();
}
properties["Data Block Count"] = Extensions.ToString(blockCnt);
long dataSize = 0;
long dataSizeUncompressed = 0;
if (blockCnt > 0)
{
for (int i = 0; i < blockCnt; ++i)
{
BCFile.BlockRegion region = reader.readerBCF.dataIndex.GetBlockRegionList()[i];
dataSize += region.GetCompressedSize();
dataSizeUncompressed += region.GetRawSize();
}
properties["Data Block Bytes"] = System.Convert.ToString(dataSize);
if (reader.readerBCF.GetDefaultCompressionName() != "none")
{
properties["Data Block Uncompressed Bytes"] = System.Convert.ToString(dataSizeUncompressed
);
properties["Data Block Compression Ratio"] = string.Format("1:%.1f", (double)dataSizeUncompressed
/ dataSize);
}
}
properties["Meta Block Count"] = Extensions.ToString(metaBlkCnt);
long metaSize = 0;
long metaSizeUncompressed = 0;
if (metaBlkCnt > 0)
{
ICollection <BCFile.MetaIndexEntry> metaBlks = reader.readerBCF.metaIndex.index.Values;
bool calculateCompression = false;
for (IEnumerator <BCFile.MetaIndexEntry> it = metaBlks.GetEnumerator(); it.HasNext
();)
{
BCFile.MetaIndexEntry e = it.Next();
metaSize += e.GetRegion().GetCompressedSize();
metaSizeUncompressed += e.GetRegion().GetRawSize();
if (e.GetCompressionAlgorithm() != Compression.Algorithm.None)
{
calculateCompression = true;
}
}
properties["Meta Block Bytes"] = System.Convert.ToString(metaSize);
if (calculateCompression)
{
properties["Meta Block Uncompressed Bytes"] = System.Convert.ToString(metaSizeUncompressed
);
properties["Meta Block Compression Ratio"] = string.Format("1:%.1f", (double)metaSizeUncompressed
/ metaSize);
}
}
properties["Meta-Data Size Ratio"] = string.Format("1:%.1f", (double)dataSize / metaSize
);
long leftOverBytes = length - dataSize - metaSize;
long miscSize = BCFile.Magic.Size() * 2 + long.Size / byte.Size + Utils.Version.Size
();
long metaIndexSize = leftOverBytes - miscSize;
properties["Meta Block Index Bytes"] = System.Convert.ToString(metaIndexSize);
properties["Headers Etc Bytes"] = System.Convert.ToString(miscSize);
// Now output the properties table.
int maxKeyLength = 0;
ICollection <KeyValuePair <string, string> > entrySet = properties;
for (IEnumerator <KeyValuePair <string, string> > it_1 = entrySet.GetEnumerator(); it_1
.HasNext();)
{
KeyValuePair <string, string> e = it_1.Next();
if (e.Key.Length > maxKeyLength)
{
maxKeyLength = e.Key.Length;
}
}
for (IEnumerator <KeyValuePair <string, string> > it_2 = entrySet.GetEnumerator(); it_2
.HasNext();)
{
KeyValuePair <string, string> e = it_2.Next();
@out.Printf("%s : %s%n", TFileDumper.Align.Format(e.Key, maxKeyLength, TFileDumper.Align
.Left), e.Value);
}
@out.WriteLine();
reader.CheckTFileDataIndex();
if (blockCnt > 0)
{
string blkID = "Data-Block";
int blkIDWidth = TFileDumper.Align.CalculateWidth(blkID, blockCnt);
int blkIDWidth2 = TFileDumper.Align.CalculateWidth(string.Empty, blockCnt);
string offset = "Offset";
int offsetWidth = TFileDumper.Align.CalculateWidth(offset, length);
string blkLen = "Length";
int blkLenWidth = TFileDumper.Align.CalculateWidth(blkLen, dataSize / blockCnt *
10);
string rawSize = "Raw-Size";
int rawSizeWidth = TFileDumper.Align.CalculateWidth(rawSize, dataSizeUncompressed
/ blockCnt * 10);
string records = "Records";
int recordsWidth = TFileDumper.Align.CalculateWidth(records, reader.GetEntryCount
() / blockCnt * 10);
string endKey = "End-Key";
int endKeyWidth = Math.Max(endKey.Length, maxKeySampleLen * 2 + 5);
@out.Printf("%s %s %s %s %s %s%n", TFileDumper.Align.Format(blkID, blkIDWidth, TFileDumper.Align
.Center), TFileDumper.Align.Format(offset, offsetWidth, TFileDumper.Align.Center
), TFileDumper.Align.Format(blkLen, blkLenWidth, TFileDumper.Align.Center), TFileDumper.Align
.Format(rawSize, rawSizeWidth, TFileDumper.Align.Center), TFileDumper.Align.Format
(records, recordsWidth, TFileDumper.Align.Center), TFileDumper.Align.Format(endKey
, endKeyWidth, TFileDumper.Align.Left));
for (int i = 0; i < blockCnt; ++i)
{
BCFile.BlockRegion region = reader.readerBCF.dataIndex.GetBlockRegionList()[i];
TFile.TFileIndexEntry indexEntry = reader.tfileIndex.GetEntry(i);
@out.Printf("%s %s %s %s %s ", TFileDumper.Align.Format(TFileDumper.Align.Format(
i, blkIDWidth2, TFileDumper.Align.ZeroPadded), blkIDWidth, TFileDumper.Align.Left
), TFileDumper.Align.Format(region.GetOffset(), offsetWidth, TFileDumper.Align.Left
), TFileDumper.Align.Format(region.GetCompressedSize(), blkLenWidth, TFileDumper.Align
.Left), TFileDumper.Align.Format(region.GetRawSize(), rawSizeWidth, TFileDumper.Align
.Left), TFileDumper.Align.Format(indexEntry.kvEntries, recordsWidth, TFileDumper.Align
.Left));
byte[] key = indexEntry.key;
bool asAscii = true;
int sampleLen = Math.Min(maxKeySampleLen, key.Length);
for (int j = 0; j < sampleLen; ++j)
{
byte b = key[j];
if ((((sbyte)b) < 32 && b != 9) || (b == 127))
{
asAscii = false;
}
}
if (!asAscii)
{
@out.Write("0X");
for (int j_1 = 0; j_1 < sampleLen; ++j_1)
{
byte b = key[i];
@out.Printf("%X", b);
}
}
else
{
@out.Write(new string(key, 0, sampleLen, Charsets.Utf8));
}
if (sampleLen < key.Length)
{
@out.Write("...");
}
@out.WriteLine();
}
}
@out.WriteLine();
if (metaBlkCnt > 0)
{
string name = "Meta-Block";
int maxNameLen = 0;
ICollection <KeyValuePair <string, BCFile.MetaIndexEntry> > metaBlkEntrySet = reader
.readerBCF.metaIndex.index;
for (IEnumerator <KeyValuePair <string, BCFile.MetaIndexEntry> > it = metaBlkEntrySet
.GetEnumerator(); it_2.HasNext();)
{
KeyValuePair <string, BCFile.MetaIndexEntry> e = it_2.Next();
if (e.Key.Length > maxNameLen)
{
maxNameLen = e.Key.Length;
}
}
int nameWidth = Math.Max(name.Length, maxNameLen);
string offset = "Offset";
int offsetWidth = TFileDumper.Align.CalculateWidth(offset, length);
string blkLen = "Length";
int blkLenWidth = TFileDumper.Align.CalculateWidth(blkLen, metaSize / metaBlkCnt
* 10);
string rawSize = "Raw-Size";
int rawSizeWidth = TFileDumper.Align.CalculateWidth(rawSize, metaSizeUncompressed
/ metaBlkCnt * 10);
string compression = "Compression";
int compressionWidth = compression.Length;
@out.Printf("%s %s %s %s %s%n", TFileDumper.Align.Format(name, nameWidth, TFileDumper.Align
.Center), TFileDumper.Align.Format(offset, offsetWidth, TFileDumper.Align.Center
), TFileDumper.Align.Format(blkLen, blkLenWidth, TFileDumper.Align.Center), TFileDumper.Align
.Format(rawSize, rawSizeWidth, TFileDumper.Align.Center), TFileDumper.Align.Format
(compression, compressionWidth, TFileDumper.Align.Left));
for (IEnumerator <KeyValuePair <string, BCFile.MetaIndexEntry> > it_3 = metaBlkEntrySet
.GetEnumerator(); it_3.HasNext();)
{
KeyValuePair <string, BCFile.MetaIndexEntry> e = it_3.Next();
string blkName = e.Value.GetMetaName();
BCFile.BlockRegion region = e.Value.GetRegion();
string blkCompression = e.Value.GetCompressionAlgorithm().GetName();
@out.Printf("%s %s %s %s %s%n", TFileDumper.Align.Format(blkName, nameWidth, TFileDumper.Align
.Left), TFileDumper.Align.Format(region.GetOffset(), offsetWidth, TFileDumper.Align
.Left), TFileDumper.Align.Format(region.GetCompressedSize(), blkLenWidth, TFileDumper.Align
.Left), TFileDumper.Align.Format(region.GetRawSize(), rawSizeWidth, TFileDumper.Align
.Left), TFileDumper.Align.Format(blkCompression, compressionWidth, TFileDumper.Align
.Left));
}
}
}
finally
{
IOUtils.Cleanup(Log, reader, fsdis);
}
}