/// <exception cref="System.IO.IOException"/>
private void ReadKeyWithoutValue(int recordIndex)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString
(ComposeSortedKey(Key, recordIndex)), null);
try
{
// read the indexed key
byte[] kbuf1 = new byte[BufSize];
int klen1 = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf1);
Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1
), ComposeSortedKey(Key, recordIndex));
if (scanner.Advance() && !scanner.AtEnd())
{
// read the next key following the indexed
byte[] kbuf2 = new byte[BufSize];
int klen2 = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf2);
Assert.Equal(Runtime.GetStringForBytes(kbuf2, 0, klen2
), ComposeSortedKey(Key, recordIndex + 1));
}
}
finally
{
scanner.Close();
reader.Close();
}
}
public virtual void TestFailureNegativeLength_2()
{
if (skip)
{
return;
}
CloseOutput();
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
scanner.LowerBound(Runtime.GetBytesForString("keyX"), 0, -1);
NUnit.Framework.Assert.Fail("Error on handling negative length.");
}
catch (Exception)
{
}
finally
{
// noop, expecting exceptions
scanner.Close();
reader.Close();
}
CloseOutput();
}
/// <exception cref="System.IO.IOException"/>
private void ReadValueBeforeKey(int recordIndex)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString
(ComposeSortedKey(Key, recordIndex)), null);
try
{
byte[] vbuf = new byte[BufSize];
int vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
Value + recordIndex);
byte[] kbuf = new byte[BufSize];
int klen = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf);
Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen),
ComposeSortedKey(Key, recordIndex));
}
finally
{
scanner.Close();
reader.Close();
}
}
// read the records and check
/// <exception cref="System.IO.IOException"/>
private int ReadAndCheckbytes(TFile.Reader.Scanner scanner, int start, int n)
{
string value = "value";
for (int i = start; i < (start + n); i++)
{
byte[] key = ReadKey(scanner);
byte[] val = ReadValue(scanner);
string keyStr = string.Format(localFormatter, i);
string valStr = value + keyStr;
Assert.True("btyes for keys do not match " + keyStr + " " + Runtime.GetStringForBytes
(key), Arrays.Equals(Runtime.GetBytesForString(keyStr), key));
Assert.True("bytes for vals do not match " + valStr + " " + Runtime.GetStringForBytes
(val), Arrays.Equals(Runtime.GetBytesForString(valStr), val));
Assert.True(scanner.Advance());
key = ReadKey(scanner);
val = ReadValue(scanner);
Assert.True("btyes for keys do not match", Arrays.Equals(Runtime.GetBytesForString
(keyStr), key));
Assert.True("bytes for vals do not match", Arrays.Equals(Runtime.GetBytesForString
(valStr), val));
Assert.True(scanner.Advance());
}
return(start + n);
}
public virtual void TestFailureReadValueManyTimes()
{
if (skip)
{
return;
}
WriteRecords(5);
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
byte[] vbuf = new byte[BufSize];
int vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
Value + 0);
try
{
scanner.Entry().GetValue(vbuf);
NUnit.Framework.Assert.Fail("Cannot get the value mlutiple times.");
}
catch (Exception)
{
}
// noop, expecting exceptions
scanner.Close();
reader.Close();
}
/// <exception cref="System.IO.IOException"/>
private int ReadPrepWithUnknownLength(TFile.Reader.Scanner scanner, int start, int
n)
{
for (int i = start; i < start; i++)
{
string key = string.Format(localFormatter, i);
byte[] read = ReadKey(scanner);
Assert.True("keys not equal", Arrays.Equals(Runtime.GetBytesForString
(key), read));
try
{
read = ReadValue(scanner);
Assert.True(false);
}
catch (IOException)
{
}
// should have thrown exception
string value = "value" + key;
read = ReadLongValue(scanner, Runtime.GetBytesForString(value).Length);
Assert.True("values nto equal", Arrays.Equals(read, Runtime.GetBytesForString
(value)));
scanner.Advance();
}
return(start + n);
}
/// <exception cref="System.IO.IOException"/>
internal virtual void ReadFile()
{
long fileLength = fs.GetFileStatus(path).GetLen();
int numSplit = 10;
long splitSize = fileLength / numSplit + 1;
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
long offset = 0;
long rowCount = 0;
BytesWritable key;
BytesWritable value;
for (int i = 0; i < numSplit; ++i, offset += splitSize)
{
TFile.Reader.Scanner scanner = reader.CreateScannerByByteRange(offset, splitSize);
int count = 0;
key = new BytesWritable();
value = new BytesWritable();
while (!scanner.AtEnd())
{
scanner.Entry().Get(key, value);
++count;
scanner.Advance();
}
scanner.Close();
Assert.True(count > 0);
rowCount += count;
}
Assert.Equal(rowCount, reader.GetEntryCount());
reader.Close();
}
// no-op
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailureNegativeOffset()
{
if (skip)
{
return;
}
WriteRecords(2, true, true);
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
byte[] buf = new byte[K];
try
{
scanner.Entry().GetKey(buf, -1);
NUnit.Framework.Assert.Fail("Failed to handle key negative offset.");
}
catch (Exception)
{
}
finally
{
}
// noop, expecting exceptions
scanner.Close();
reader.Close();
}
/// <exception cref="System.IO.IOException"/>
internal static void ReadRecords(FileSystem fs, Path path, int count, Configuration
conf)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
for (int nx = 0; nx < count; nx++, scanner.Advance())
{
NUnit.Framework.Assert.IsFalse(scanner.AtEnd());
// Assert.assertTrue(scanner.next());
byte[] kbuf = new byte[BufSize];
int klen = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf);
Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen),
ComposeSortedKey(Key, nx));
byte[] vbuf = new byte[BufSize];
int vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
Value + nx);
}
Assert.True(scanner.AtEnd());
NUnit.Framework.Assert.IsFalse(scanner.Advance());
}
finally
{
scanner.Close();
reader.Close();
}
}
// read a value from the scanner
/// <exception cref="System.IO.IOException"/>
public virtual byte[] ReadValue(TFile.Reader.Scanner scanner)
{
int valueLen = scanner.Entry().GetValueLength();
byte[] read = new byte[valueLen];
scanner.Entry().GetValue(read);
return(read);
}
// do nothing
// read a key from the scanner
/// <exception cref="System.IO.IOException"/>
public virtual byte[] ReadKey(TFile.Reader.Scanner scanner)
{
int keylen = scanner.Entry().GetKeyLength();
byte[] read = new byte[keylen];
scanner.Entry().GetKey(read);
return(read);
}
// read a long value from the scanner
/// <exception cref="System.IO.IOException"/>
public virtual byte[] ReadLongValue(TFile.Reader.Scanner scanner, int len)
{
DataInputStream din = scanner.Entry().GetValueStream();
byte[] b = new byte[len];
din.ReadFully(b);
din.Close();
return(b);
}
/// <exception cref="System.IO.IOException"/>
internal virtual TFile.Reader.Location Locate(TFile.Reader.Scanner scanner, byte[]
key)
{
if (scanner.SeekTo(key) == true)
{
return(scanner.currentLocation);
}
return(scanner.endLocation);
}
/// <exception cref="System.IO.IOException"/>
public LogReader(Configuration conf, Path remoteAppLogFile)
{
FileContext fileContext = FileContext.GetFileContext(conf);
this.fsDataIStream = fileContext.Open(remoteAppLogFile);
reader = new TFile.Reader(this.fsDataIStream, fileContext.GetFileStatus(remoteAppLogFile
).GetLen(), conf);
this.scanner = reader.CreateScanner();
}
/// <exception cref="System.IO.IOException"/>
private void ReadAllRecords(TFile.Reader.Scanner scanner)
{
ReadEmptyRecords(scanner, 10);
int ret = ReadAndCheckbytes(scanner, 0, 100);
ret = ReadLargeRecords(scanner, ret, 1);
ret = ReadPrepWithKnownLength(scanner, ret, 40);
ret = ReadPrepWithUnknownLength(scanner, ret, 50);
}
/// <exception cref="System.IO.IOException"/>
public TFileReadable(FileSystem fs, Path path, int osBufferSize, Configuration conf
)
{
this.fsdis = fs.Open(path, osBufferSize);
this.reader = new TFile.Reader(fsdis, fs.GetFileStatus(path).GetLen(), conf);
this.scanner = reader.CreateScanner();
keyBuffer = new byte[32];
valueBuffer = new byte[32];
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailureSeek()
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
// can't find ceil
try
{
scanner.LowerBound(Runtime.GetBytesForString("keyN"));
NUnit.Framework.Assert.Fail("Cannot search in a unsorted TFile!");
}
catch (Exception)
{
}
finally
{
}
// noop, expecting excetions
// can't find higher
try
{
scanner.UpperBound(Runtime.GetBytesForString("keyA"));
NUnit.Framework.Assert.Fail("Cannot search higher in a unsorted TFile!");
}
catch (Exception)
{
}
finally
{
}
// noop, expecting excetions
// can't seek
try
{
scanner.SeekTo(Runtime.GetBytesForString("keyM"));
NUnit.Framework.Assert.Fail("Cannot search a unsorted TFile!");
}
catch (Exception)
{
}
finally
{
}
}
finally
{
// noop, expecting excetions
scanner.Close();
reader.Close();
}
}
/// <exception cref="System.IO.IOException"/>
private void CheckBlockIndex(int recordIndex, int blockIndexExpected)
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
scanner.SeekTo(Runtime.GetBytesForString(ComposeSortedKey(Key, recordIndex
)));
Assert.Equal(blockIndexExpected, scanner.currentLocation.GetBlockIndex
());
scanner.Close();
reader.Close();
}
/// <summary>test none codecs</summary>
/// <exception cref="System.IO.IOException"/>
internal virtual void BasicWithSomeCodec(string codec)
{
Path ncTFile = new Path(Root, "basic.tfile");
FSDataOutputStream fout = CreateFSOutput(ncTFile);
TFile.Writer writer = new TFile.Writer(fout, minBlockSize, codec, "memcmp", conf);
WriteRecords(writer);
fout.Close();
FSDataInputStream fin = fs.Open(ncTFile);
TFile.Reader reader = new TFile.Reader(fs.Open(ncTFile), fs.GetFileStatus(ncTFile
).GetLen(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
ReadAllRecords(scanner);
scanner.SeekTo(GetSomeKey(50));
Assert.True("location lookup failed", scanner.SeekTo(GetSomeKey
(50)));
// read the key and see if it matches
byte[] readKey = ReadKey(scanner);
Assert.True("seeked key does not match", Arrays.Equals(GetSomeKey
(50), readKey));
scanner.SeekTo(new byte[0]);
byte[] val1 = ReadValue(scanner);
scanner.SeekTo(new byte[0]);
byte[] val2 = ReadValue(scanner);
Assert.True(Arrays.Equals(val1, val2));
// check for lowerBound
scanner.LowerBound(GetSomeKey(50));
Assert.True("locaton lookup failed", scanner.currentLocation.CompareTo
(reader.End()) < 0);
readKey = ReadKey(scanner);
Assert.True("seeked key does not match", Arrays.Equals(readKey,
GetSomeKey(50)));
// check for upper bound
scanner.UpperBound(GetSomeKey(50));
Assert.True("location lookup failed", scanner.currentLocation.CompareTo
(reader.End()) < 0);
readKey = ReadKey(scanner);
Assert.True("seeked key does not match", Arrays.Equals(readKey,
GetSomeKey(51)));
scanner.Close();
// test for a range of scanner
scanner = reader.CreateScannerByKey(GetSomeKey(10), GetSomeKey(60));
ReadAndCheckbytes(scanner, 10, 50);
NUnit.Framework.Assert.IsFalse(scanner.Advance());
scanner.Close();
reader.Close();
fin.Close();
fs.Delete(ncTFile, true);
}
public virtual void TestNoDataEntry()
{
if (skip)
{
return;
}
CloseOutput();
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
Assert.True(reader.IsSorted());
TFile.Reader.Scanner scanner = reader.CreateScanner();
Assert.True(scanner.AtEnd());
scanner.Close();
reader.Close();
}
// read large records
// read them twice since its duplicated
/// <exception cref="System.IO.IOException"/>
private int ReadLargeRecords(TFile.Reader.Scanner scanner, int start, int n)
{
for (int i = start; i < (start + n); i++)
{
byte[] key = ReadKey(scanner);
string keyStr = string.Format(localFormatter, i);
Assert.True("btyes for keys do not match", Arrays.Equals(Runtime.GetBytesForString
(keyStr), key));
scanner.Advance();
key = ReadKey(scanner);
Assert.True("btyes for keys do not match", Arrays.Equals(Runtime.GetBytesForString
(keyStr), key));
scanner.Advance();
}
return(start + n);
}
/// <summary>Returns the owner of the application.</summary>
/// <returns>the application owner.</returns>
/// <exception cref="System.IO.IOException"/>
public virtual string GetApplicationOwner()
{
TFile.Reader.Scanner ownerScanner = reader.CreateScanner();
AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey();
while (!ownerScanner.AtEnd())
{
TFile.Reader.Scanner.Entry entry = ownerScanner.Entry();
key.ReadFields(entry.GetKeyStream());
if (key.ToString().Equals(ApplicationOwnerKey.ToString()))
{
DataInputStream valueStream = entry.GetValueStream();
return(valueStream.ReadUTF());
}
ownerScanner.Advance();
}
return(null);
}
// read empty keys and values
/// <exception cref="System.IO.IOException"/>
private void ReadEmptyRecords(TFile.Reader.Scanner scanner, int n)
{
byte[] key = new byte[0];
byte[] value = new byte[0];
byte[] readKey = null;
byte[] readValue = null;
for (int i = 0; i < n; i++)
{
readKey = ReadKey(scanner);
readValue = ReadValue(scanner);
Assert.True("failed to match keys", Arrays.Equals(readKey, key)
);
Assert.True("failed to match values", Arrays.Equals(readValue,
value));
Assert.True("failed to advance cursor", scanner.Advance());
}
}
/// <exception cref="System.IO.IOException"/>
private int ReadPrepWithKnownLength(TFile.Reader.Scanner scanner, int start, int
n)
{
for (int i = start; i < (start + n); i++)
{
string key = string.Format(localFormatter, i);
byte[] read = ReadKey(scanner);
Assert.True("keys not equal", Arrays.Equals(Runtime.GetBytesForString
(key), read));
string value = "value" + key;
read = ReadValue(scanner);
Assert.True("values not equal", Arrays.Equals(Runtime.GetBytesForString
(value), read));
scanner.Advance();
}
return(start + n);
}
/// <summary>Returns ACLs for the application.</summary>
/// <remarks>
/// Returns ACLs for the application. An empty map is returned if no ACLs are
/// found.
/// </remarks>
/// <returns>a map of the Application ACLs.</returns>
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ApplicationAccessType, string> GetApplicationAcls()
{
// TODO Seek directly to the key once a comparator is specified.
TFile.Reader.Scanner aclScanner = reader.CreateScanner();
AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey();
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>();
while (!aclScanner.AtEnd())
{
TFile.Reader.Scanner.Entry entry = aclScanner.Entry();
key.ReadFields(entry.GetKeyStream());
if (key.ToString().Equals(ApplicationAclKey.ToString()))
{
DataInputStream valueStream = entry.GetValueStream();
while (true)
{
string appAccessOp = null;
string aclString = null;
try
{
appAccessOp = valueStream.ReadUTF();
}
catch (EOFException)
{
// Valid end of stream.
break;
}
try
{
aclString = valueStream.ReadUTF();
}
catch (EOFException e)
{
throw new YarnRuntimeException("Error reading ACLs", e);
}
acls[ApplicationAccessType.ValueOf(appAccessOp)] = aclString;
}
}
aclScanner.Advance();
}
return(acls);
}
// unsorted with some codec
/// <exception cref="System.IO.IOException"/>
internal virtual void UnsortedWithSomeCodec(string codec)
{
Path uTfile = new Path(Root, "unsorted.tfile");
FSDataOutputStream fout = CreateFSOutput(uTfile);
TFile.Writer writer = new TFile.Writer(fout, minBlockSize, codec, null, conf);
WriteRecords(writer);
writer.Close();
fout.Close();
FSDataInputStream fin = fs.Open(uTfile);
TFile.Reader reader = new TFile.Reader(fs.Open(uTfile), fs.GetFileStatus(uTfile).
GetLen(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
ReadAllRecords(scanner);
scanner.Close();
reader.Close();
fin.Close();
fs.Delete(uTfile, true);
}
/// <exception cref="System.IO.IOException"/>
public virtual void SeekTFile()
{
int miss = 0;
long totalBytes = 0;
FSDataInputStream fsdis = fs.Open(path);
TFile.Reader reader = new TFile.Reader(fsdis, fs.GetFileStatus(path).GetLen(), conf
);
KeySampler kSampler = new KeySampler(rng, reader.GetFirstKey(), reader.GetLastKey
(), keyLenGen);
TFile.Reader.Scanner scanner = reader.CreateScanner();
BytesWritable key = new BytesWritable();
BytesWritable val = new BytesWritable();
timer.Reset();
timer.Start();
for (int i = 0; i < options.seekCount; ++i)
{
kSampler.Next(key);
scanner.LowerBound(key.Get(), 0, key.GetSize());
if (!scanner.AtEnd())
{
scanner.Entry().Get(key, val);
totalBytes += key.GetSize();
totalBytes += val.GetSize();
}
else
{
++miss;
}
}
timer.Stop();
double duration = (double)timer.Read() / 1000;
// in us.
System.Console.Out.Printf("time: %s...avg seek: %s...%d hit...%d miss...avg I/O size: %.2fKB\n"
, timer.ToString(), NanoTimer.NanoTimeToString(timer.Read() / options.seekCount)
, options.seekCount - miss, miss, (double)totalBytes / 1024 / (options.seekCount
- miss));
}
// we still can scan records in an unsorted TFile
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailureScannerWithKeys()
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
NUnit.Framework.Assert.IsFalse(reader.IsSorted());
Assert.Equal((int)reader.GetEntryCount(), 4);
try
{
TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString
("aaa"), Runtime.GetBytesForString("zzz"));
NUnit.Framework.Assert.Fail("Failed to catch creating scanner with keys on unsorted file."
);
}
catch (RuntimeException)
{
}
finally
{
reader.Close();
}
}
public virtual void TestLocate()
{
if (skip)
{
return;
}
WriteRecords(3 * records1stBlock);
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, 2)));
Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, records1stBlock
- 1)));
Locate(scanner, Runtime.GetBytesForString(ComposeSortedKey(Key, records1stBlock
)));
TFile.Reader.Location locX = Locate(scanner, Runtime.GetBytesForString("keyX"
));
Assert.Equal(scanner.endLocation, locX);
scanner.Close();
reader.Close();
}
// we still can scan records in an unsorted TFile
/// <exception cref="System.IO.IOException"/>
public virtual void TestScan()
{
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
NUnit.Framework.Assert.IsFalse(reader.IsSorted());
Assert.Equal((int)reader.GetEntryCount(), 4);
TFile.Reader.Scanner scanner = reader.CreateScanner();
try
{
// read key and value
byte[] kbuf = new byte[BufSize];
int klen = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf);
Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen),
"keyZ");
byte[] vbuf = new byte[BufSize];
int vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
"valueZ");
scanner.Advance();
// now try get value first
vbuf = new byte[BufSize];
vlen = scanner.Entry().GetValueLength();
scanner.Entry().GetValue(vbuf);
Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen),
"valueM");
kbuf = new byte[BufSize];
klen = scanner.Entry().GetKeyLength();
scanner.Entry().GetKey(kbuf);
Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen),
"keyM");
}
finally
{
scanner.Close();
reader.Close();
}
}
