/// <summary>test none codecs</summary>
/// <exception cref="System.IO.IOException"/>
internal virtual void BasicWithSomeCodec(string codec)
{
Path ncTFile = new Path(Root, "basic.tfile");
FSDataOutputStream fout = CreateFSOutput(ncTFile);
TFile.Writer writer = new TFile.Writer(fout, minBlockSize, codec, "memcmp", conf);
WriteRecords(writer);
fout.Close();
FSDataInputStream fin = fs.Open(ncTFile);
TFile.Reader reader = new TFile.Reader(fs.Open(ncTFile), fs.GetFileStatus(ncTFile
).GetLen(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
ReadAllRecords(scanner);
scanner.SeekTo(GetSomeKey(50));
Assert.True("location lookup failed", scanner.SeekTo(GetSomeKey
(50)));
// read the key and see if it matches
byte[] readKey = ReadKey(scanner);
Assert.True("seeked key does not match", Arrays.Equals(GetSomeKey
(50), readKey));
scanner.SeekTo(new byte[0]);
byte[] val1 = ReadValue(scanner);
scanner.SeekTo(new byte[0]);
byte[] val2 = ReadValue(scanner);
Assert.True(Arrays.Equals(val1, val2));
// check for lowerBound
scanner.LowerBound(GetSomeKey(50));
Assert.True("locaton lookup failed", scanner.currentLocation.CompareTo
(reader.End()) < 0);
readKey = ReadKey(scanner);
Assert.True("seeked key does not match", Arrays.Equals(readKey,
GetSomeKey(50)));
// check for upper bound
scanner.UpperBound(GetSomeKey(50));
Assert.True("location lookup failed", scanner.currentLocation.CompareTo
(reader.End()) < 0);
readKey = ReadKey(scanner);
Assert.True("seeked key does not match", Arrays.Equals(readKey,
GetSomeKey(51)));
scanner.Close();
// test for a range of scanner
scanner = reader.CreateScannerByKey(GetSomeKey(10), GetSomeKey(60));
ReadAndCheckbytes(scanner, 10, 50);
NUnit.Framework.Assert.IsFalse(scanner.Advance());
scanner.Close();
reader.Close();
fin.Close();
fs.Delete(ncTFile, true);
}
/// <exception cref="System.IO.IOException"/>
private void ReadValueBeforeKey(int recordIndex)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString
(ComposeSortedKey(Key, recordIndex)), null);
try
{
byte[] vbuf = new byte[BufSize];
int vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
Value + recordIndex);
byte[] kbuf = new byte[BufSize];
int klen = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf);
Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen),
ComposeSortedKey(Key, recordIndex));
}
finally
{
scanner.Close();
reader.Close();
}
}
// no-op
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailureNegativeOffset()
{
if (skip)
{
return;
}
WriteRecords(2, true, true);
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
byte[] buf = new byte[K];
try
{
scanner.Entry().GetKey(buf, -1);
NUnit.Framework.Assert.Fail("Failed to handle key negative offset.");
}
catch (Exception)
{
}
finally
{
}
// noop, expecting exceptions
scanner.Close();
reader.Close();
}
/// <exception cref="System.IO.IOException"/>
private void ReadKeyWithoutValue(int recordIndex)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString
(ComposeSortedKey(Key, recordIndex)), null);
try
{
// read the indexed key
byte[] kbuf1 = new byte[BufSize];
int klen1 = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf1);
Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1
), ComposeSortedKey(Key, recordIndex));
if (scanner.Advance() && !scanner.AtEnd())
{
// read the next key following the indexed
byte[] kbuf2 = new byte[BufSize];
int klen2 = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf2);
Assert.Equal(Runtime.GetStringForBytes(kbuf2, 0, klen2
), ComposeSortedKey(Key, recordIndex + 1));
}
}
finally
{
scanner.Close();
reader.Close();
}
}
/// <exception cref="System.IO.IOException"/>
internal static void ReadRecords(FileSystem fs, Path path, int count, Configuration
conf)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
for (int nx = 0; nx < count; nx++, scanner.Advance())
{
NUnit.Framework.Assert.IsFalse(scanner.AtEnd());
// Assert.assertTrue(scanner.next());
byte[] kbuf = new byte[BufSize];
int klen = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf);
Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen),
ComposeSortedKey(Key, nx));
byte[] vbuf = new byte[BufSize];
int vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
Value + nx);
}
Assert.True(scanner.AtEnd());
NUnit.Framework.Assert.IsFalse(scanner.Advance());
}
finally
{
scanner.Close();
reader.Close();
}
}
/// <exception cref="System.IO.IOException"/>
internal virtual void ReadFile()
{
long fileLength = fs.GetFileStatus(path).GetLen();
int numSplit = 10;
long splitSize = fileLength / numSplit + 1;
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
long offset = 0;
long rowCount = 0;
BytesWritable key;
BytesWritable value;
for (int i = 0; i < numSplit; ++i, offset += splitSize)
{
TFile.Reader.Scanner scanner = reader.CreateScannerByByteRange(offset, splitSize);
int count = 0;
key = new BytesWritable();
value = new BytesWritable();
while (!scanner.AtEnd())
{
scanner.Entry().Get(key, value);
++count;
scanner.Advance();
}
scanner.Close();
Assert.True(count > 0);
rowCount += count;
}
Assert.Equal(rowCount, reader.GetEntryCount());
reader.Close();
}
public virtual void TestFailureNegativeLength_2()
{
if (skip)
{
return;
}
CloseOutput();
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
scanner.LowerBound(Runtime.GetBytesForString("keyX"), 0, -1);
NUnit.Framework.Assert.Fail("Error on handling negative length.");
}
catch (Exception)
{
}
finally
{
// noop, expecting exceptions
scanner.Close();
reader.Close();
}
CloseOutput();
}
public virtual void TestFailureReadValueManyTimes()
{
if (skip)
{
return;
}
WriteRecords(5);
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
byte[] vbuf = new byte[BufSize];
int vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
Value + 0);
try
{
scanner.Entry().GetValue(vbuf);
NUnit.Framework.Assert.Fail("Cannot get the value mlutiple times.");
}
catch (Exception)
{
}
// noop, expecting exceptions
scanner.Close();
reader.Close();
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailureSeek()
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
// can't find ceil
try
{
scanner.LowerBound(Runtime.GetBytesForString("keyN"));
NUnit.Framework.Assert.Fail("Cannot search in a unsorted TFile!");
}
catch (Exception)
{
}
finally
{
}
// noop, expecting excetions
// can't find higher
try
{
scanner.UpperBound(Runtime.GetBytesForString("keyA"));
NUnit.Framework.Assert.Fail("Cannot search higher in a unsorted TFile!");
}
catch (Exception)
{
}
finally
{
}
// noop, expecting excetions
// can't seek
try
{
scanner.SeekTo(Runtime.GetBytesForString("keyM"));
NUnit.Framework.Assert.Fail("Cannot search a unsorted TFile!");
}
catch (Exception)
{
}
finally
{
}
}
finally
{
// noop, expecting excetions
scanner.Close();
reader.Close();
}
}
/// <exception cref="System.IO.IOException"/>
private void CheckBlockIndex(int recordIndex, int blockIndexExpected)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
scanner.SeekTo(Runtime.GetBytesForString(ComposeSortedKey(Key, recordIndex
)));
Assert.Equal(blockIndexExpected, scanner.currentLocation.GetBlockIndex
());
scanner.Close();
reader.Close();
}
public virtual void TestNoDataEntry()
{
if (skip)
{
return;
}
CloseOutput();
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
Assert.True(reader.IsSorted());
TFile.Reader.Scanner scanner = reader.CreateScanner();
Assert.True(scanner.AtEnd());
scanner.Close();
reader.Close();
}
// unsorted with some codec
/// <exception cref="System.IO.IOException"/>
internal virtual void UnsortedWithSomeCodec(string codec)
{
Path uTfile = new Path(Root, "unsorted.tfile");
FSDataOutputStream fout = CreateFSOutput(uTfile);
TFile.Writer writer = new TFile.Writer(fout, minBlockSize, codec, null, conf);
WriteRecords(writer);
writer.Close();
fout.Close();
FSDataInputStream fin = fs.Open(uTfile);
TFile.Reader reader = new TFile.Reader(fs.Open(uTfile), fs.GetFileStatus(uTfile).
GetLen(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
ReadAllRecords(scanner);
scanner.Close();
reader.Close();
fin.Close();
fs.Delete(uTfile, true);
}
public virtual void TestLocate()
{
if (skip)
{
return;
}
WriteRecords(3 * records1stBlock);
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, 2)));
Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, records1stBlock
- 1)));
Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, records1stBlock
)));
TFile.Reader.Location locX = Locate(scanner, Runtime.GetBytesForString("keyX"
));
Assert.Equal(scanner.endLocation, locX);
scanner.Close();
reader.Close();
}
// we still can scan records in an unsorted TFile
/// <exception cref="System.IO.IOException"/>
public virtual void TestScan()
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
NUnit.Framework.Assert.IsFalse(reader.IsSorted());
Assert.Equal((int)reader.GetEntryCount(), 4);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
// read key and value
byte[] kbuf = new byte[BufSize];
int klen = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf);
Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen),
"keyZ");
byte[] vbuf = new byte[BufSize];
int vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
"valueZ");
scanner.Advance();
// now try get value first
vbuf = new byte[BufSize];
vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
"valueM");
kbuf = new byte[BufSize];
klen = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf);
Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen),
"keyM");
}
finally
{
scanner.Close();
reader.Close();
}
}
public virtual void TestFailureNegativeLength_3()
{
if (skip)
{
return;
}
WriteRecords(3);
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
// test negative array offset
try
{
scanner.SeekTo(Runtime.GetBytesForString("keyY"), -1, 4);
NUnit.Framework.Assert.Fail("Failed to handle negative offset.");
}
catch (Exception)
{
}
// noop, expecting exceptions
// test negative array length
try
{
scanner.SeekTo(Runtime.GetBytesForString("keyY"), 0, -2);
NUnit.Framework.Assert.Fail("Failed to handle negative key length.");
}
catch (Exception)
{
}
}
finally
{
// noop, expecting exceptions
reader.Close();
scanner.Close();
}
}
/* Similar to readFile(), tests the scanner created
* by record numbers rather than the offsets.
*/
/// <exception cref="System.IO.IOException"/>
internal virtual void ReadRowSplits(int numSplits)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
long totalRecords = reader.GetEntryCount();
for (int i = 0; i < numSplits; i++)
{
long startRec = i * totalRecords / numSplits;
long endRec = (i + 1) * totalRecords / numSplits;
if (i == numSplits - 1)
{
endRec = totalRecords;
}
TFile.Reader.Scanner scanner = reader.CreateScannerByRecordNum(startRec, endRec);
int count = 0;
BytesWritable key = new BytesWritable();
BytesWritable value = new BytesWritable();
long x = startRec;
while (!scanner.AtEnd())
{
Assert.Equal("Incorrect RecNum returned by scanner", scanner.GetRecordNum
(), x);
scanner.Entry().Get(key, value);
++count;
Assert.Equal("Incorrect RecNum returned by scanner", scanner.GetRecordNum
(), x);
scanner.Advance();
++x;
}
scanner.Close();
Assert.True(count == (endRec - startRec));
}
// make sure specifying range at the end gives zero records.
TFile.Reader.Scanner scanner_1 = reader.CreateScannerByRecordNum(totalRecords, -1
);
Assert.True(scanner_1.AtEnd());
}
/// <exception cref="System.IO.IOException"/>
private void ReadValueWithoutKey(int recordIndex)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString
(ComposeSortedKey(Key, recordIndex)), null);
byte[] vbuf1 = new byte[BufSize];
int vlen1 = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf1);
Assert.Equal(Runtime.GetStringForBytes(vbuf1, 0, vlen1
), Value + recordIndex);
if (scanner.Advance() && !scanner.AtEnd())
{
byte[] vbuf2 = new byte[BufSize];
int vlen2 = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf2);
Assert.Equal(Runtime.GetStringForBytes(vbuf2, 0, vlen2
), Value + (recordIndex + 1));
}
scanner.Close();
reader.Close();
}
/// <exception cref="System.IO.IOException"/>
private void ReadKeyManyTimes(int recordIndex)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString
(ComposeSortedKey(Key, recordIndex)), null);
// read the indexed key
byte[] kbuf1 = new byte[BufSize];
int klen1 = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf1);
Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1
), ComposeSortedKey(Key, recordIndex));
klen1 = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf1);
Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1
), ComposeSortedKey(Key, recordIndex));
klen1 = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf1);
Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1
), ComposeSortedKey(Key, recordIndex));
scanner.Close();
reader.Close();
}
/// <exception cref="System.IO.IOException"/>
public virtual void Close()
{
scanner.Close();
reader.Close();
fsdis.Close();
}
