public virtual void TestFailureReadValueManyTimes() { if (skip) { return; } WriteRecords(5); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); byte[] vbuf = new byte[BufSize]; int vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), Value + 0); try { scanner.Entry().GetValue(vbuf); NUnit.Framework.Assert.Fail("Cannot get the value mlutiple times."); } catch (Exception) { } // noop, expecting exceptions scanner.Close(); reader.Close(); }
/// <exception cref="System.IO.IOException"/> internal static void ReadRecords(FileSystem fs, Path path, int count, Configuration conf) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); try { for (int nx = 0; nx < count; nx++, scanner.Advance()) { NUnit.Framework.Assert.IsFalse(scanner.AtEnd()); // Assert.assertTrue(scanner.next()); byte[] kbuf = new byte[BufSize]; int klen = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf); Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen), ComposeSortedKey(Key, nx)); byte[] vbuf = new byte[BufSize]; int vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), Value + nx); } Assert.True(scanner.AtEnd()); NUnit.Framework.Assert.IsFalse(scanner.Advance()); } finally { scanner.Close(); reader.Close(); } }
/// <exception cref="System.IO.IOException"/> private void ReadKeyWithoutValue(int recordIndex) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString (ComposeSortedKey(Key, recordIndex)), null); try { // read the indexed key byte[] kbuf1 = new byte[BufSize]; int klen1 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf1); Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1 ), ComposeSortedKey(Key, recordIndex)); if (scanner.Advance() && !scanner.AtEnd()) { // read the next key following the indexed byte[] kbuf2 = new byte[BufSize]; int klen2 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf2); Assert.Equal(Runtime.GetStringForBytes(kbuf2, 0, klen2 ), ComposeSortedKey(Key, recordIndex + 1)); } } finally { scanner.Close(); reader.Close(); } }
/// <exception cref="System.IO.IOException"/> private void ReadValueBeforeKey(int recordIndex) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString (ComposeSortedKey(Key, recordIndex)), null); try { byte[] vbuf = new byte[BufSize]; int vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), Value + recordIndex); byte[] kbuf = new byte[BufSize]; int klen = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf); Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen), ComposeSortedKey(Key, recordIndex)); } finally { scanner.Close(); reader.Close(); } }
// do nothing // read a key from the scanner /// <exception cref="System.IO.IOException"/> public virtual byte[] ReadKey(TFile.Reader.Scanner scanner) { int keylen = scanner.Entry().GetKeyLength(); byte[] read = new byte[keylen]; scanner.Entry().GetKey(read); return(read); }
// read a value from the scanner /// <exception cref="System.IO.IOException"/> public virtual byte[] ReadValue(TFile.Reader.Scanner scanner) { int valueLen = scanner.Entry().GetValueLength(); byte[] read = new byte[valueLen]; scanner.Entry().GetValue(read); return(read); }
// no-op /// <exception cref="System.IO.IOException"/> public virtual void TestFailureNegativeOffset() { if (skip) { return; } WriteRecords(2, true, true); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScanner(); byte[] buf = new byte[K]; try { scanner.Entry().GetKey(buf, -1); NUnit.Framework.Assert.Fail("Failed to handle key negative offset."); } catch (Exception) { } finally { } // noop, expecting exceptions scanner.Close(); reader.Close(); }
/// <exception cref="System.IO.IOException"/> internal virtual void ReadFile() { long fileLength = fs.GetFileStatus(path).GetLen(); int numSplit = 10; long splitSize = fileLength / numSplit + 1; TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); long offset = 0; long rowCount = 0; BytesWritable key; BytesWritable value; for (int i = 0; i < numSplit; ++i, offset += splitSize) { TFile.Reader.Scanner scanner = reader.CreateScannerByByteRange(offset, splitSize); int count = 0; key = new BytesWritable(); value = new BytesWritable(); while (!scanner.AtEnd()) { scanner.Entry().Get(key, value); ++count; scanner.Advance(); } scanner.Close(); Assert.True(count > 0); rowCount += count; } Assert.Equal(rowCount, reader.GetEntryCount()); reader.Close(); }
// read a long value from the scanner /// <exception cref="System.IO.IOException"/> public virtual byte[] ReadLongValue(TFile.Reader.Scanner scanner, int len) { DataInputStream din = scanner.Entry().GetValueStream(); byte[] b = new byte[len]; din.ReadFully(b); din.Close(); return(b); }
/// <exception cref="System.IO.IOException"/> private void ReadValueWithoutKey(int recordIndex) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString (ComposeSortedKey(Key, recordIndex)), null); byte[] vbuf1 = new byte[BufSize]; int vlen1 = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf1); Assert.Equal(Runtime.GetStringForBytes(vbuf1, 0, vlen1 ), Value + recordIndex); if (scanner.Advance() && !scanner.AtEnd()) { byte[] vbuf2 = new byte[BufSize]; int vlen2 = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf2); Assert.Equal(Runtime.GetStringForBytes(vbuf2, 0, vlen2 ), Value + (recordIndex + 1)); } scanner.Close(); reader.Close(); }
/// <exception cref="System.IO.IOException"/> public virtual bool Next() { if (scanner.AtEnd()) { return(false); } TFile.Reader.Scanner.Entry entry = scanner.Entry(); keyLength = entry.GetKeyLength(); CheckKeyBuffer(keyLength); entry.GetKey(keyBuffer); valueLength = entry.GetValueLength(); CheckValueBuffer(valueLength); entry.GetValue(valueBuffer); scanner.Advance(); return(true); }
/// <summary>Returns the owner of the application.</summary> /// <returns>the application owner.</returns> /// <exception cref="System.IO.IOException"/> public virtual string GetApplicationOwner() { TFile.Reader.Scanner ownerScanner = reader.CreateScanner(); AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey(); while (!ownerScanner.AtEnd()) { TFile.Reader.Scanner.Entry entry = ownerScanner.Entry(); key.ReadFields(entry.GetKeyStream()); if (key.ToString().Equals(ApplicationOwnerKey.ToString())) { DataInputStream valueStream = entry.GetValueStream(); return(valueStream.ReadUTF()); } ownerScanner.Advance(); } return(null); }
/// <summary>Returns ACLs for the application.</summary> /// <remarks> /// Returns ACLs for the application. An empty map is returned if no ACLs are /// found. /// </remarks> /// <returns>a map of the Application ACLs.</returns> /// <exception cref="System.IO.IOException"/> public virtual IDictionary <ApplicationAccessType, string> GetApplicationAcls() { // TODO Seek directly to the key once a comparator is specified. TFile.Reader.Scanner aclScanner = reader.CreateScanner(); AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey(); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(); while (!aclScanner.AtEnd()) { TFile.Reader.Scanner.Entry entry = aclScanner.Entry(); key.ReadFields(entry.GetKeyStream()); if (key.ToString().Equals(ApplicationAclKey.ToString())) { DataInputStream valueStream = entry.GetValueStream(); while (true) { string appAccessOp = null; string aclString = null; try { appAccessOp = valueStream.ReadUTF(); } catch (EOFException) { // Valid end of stream. break; } try { aclString = valueStream.ReadUTF(); } catch (EOFException e) { throw new YarnRuntimeException("Error reading ACLs", e); } acls[ApplicationAccessType.ValueOf(appAccessOp)] = aclString; } } aclScanner.Advance(); } return(acls); }
// we still can scan records in an unsorted TFile /// <exception cref="System.IO.IOException"/> public virtual void TestScan() { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); NUnit.Framework.Assert.IsFalse(reader.IsSorted()); Assert.Equal((int)reader.GetEntryCount(), 4); TFile.Reader.Scanner scanner = reader.CreateScanner(); try { // read key and value byte[] kbuf = new byte[BufSize]; int klen = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf); Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen), "keyZ"); byte[] vbuf = new byte[BufSize]; int vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), "valueZ"); scanner.Advance(); // now try get value first vbuf = new byte[BufSize]; vlen = scanner.Entry().GetValueLength(); scanner.Entry().GetValue(vbuf); Assert.Equal(Runtime.GetStringForBytes(vbuf, 0, vlen), "valueM"); kbuf = new byte[BufSize]; klen = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf); Assert.Equal(Runtime.GetStringForBytes(kbuf, 0, klen), "keyM"); } finally { scanner.Close(); reader.Close(); } }
/// <exception cref="System.IO.IOException"/> public virtual void SeekTFile() { int miss = 0; long totalBytes = 0; FSDataInputStream fsdis = fs.Open(path); TFile.Reader reader = new TFile.Reader(fsdis, fs.GetFileStatus(path).GetLen(), conf ); KeySampler kSampler = new KeySampler(rng, reader.GetFirstKey(), reader.GetLastKey (), keyLenGen); TFile.Reader.Scanner scanner = reader.CreateScanner(); BytesWritable key = new BytesWritable(); BytesWritable val = new BytesWritable(); timer.Reset(); timer.Start(); for (int i = 0; i < options.seekCount; ++i) { kSampler.Next(key); scanner.LowerBound(key.Get(), 0, key.GetSize()); if (!scanner.AtEnd()) { scanner.Entry().Get(key, val); totalBytes += key.GetSize(); totalBytes += val.GetSize(); } else { ++miss; } } timer.Stop(); double duration = (double)timer.Read() / 1000; // in us. System.Console.Out.Printf("time: %s...avg seek: %s...%d hit...%d miss...avg I/O size: %.2fKB\n" , timer.ToString(), NanoTimer.NanoTimeToString(timer.Read() / options.seekCount) , options.seekCount - miss, miss, (double)totalBytes / 1024 / (options.seekCount - miss)); }
/* Similar to readFile(), tests the scanner created * by record numbers rather than the offsets. */ /// <exception cref="System.IO.IOException"/> internal virtual void ReadRowSplits(int numSplits) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); long totalRecords = reader.GetEntryCount(); for (int i = 0; i < numSplits; i++) { long startRec = i * totalRecords / numSplits; long endRec = (i + 1) * totalRecords / numSplits; if (i == numSplits - 1) { endRec = totalRecords; } TFile.Reader.Scanner scanner = reader.CreateScannerByRecordNum(startRec, endRec); int count = 0; BytesWritable key = new BytesWritable(); BytesWritable value = new BytesWritable(); long x = startRec; while (!scanner.AtEnd()) { Assert.Equal("Incorrect RecNum returned by scanner", scanner.GetRecordNum (), x); scanner.Entry().Get(key, value); ++count; Assert.Equal("Incorrect RecNum returned by scanner", scanner.GetRecordNum (), x); scanner.Advance(); ++x; } scanner.Close(); Assert.True(count == (endRec - startRec)); } // make sure specifying range at the end gives zero records. TFile.Reader.Scanner scanner_1 = reader.CreateScannerByRecordNum(totalRecords, -1 ); Assert.True(scanner_1.AtEnd()); }
/// <exception cref="System.IO.IOException"/> private void ReadKeyManyTimes(int recordIndex) { TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); TFile.Reader.Scanner scanner = reader.CreateScannerByKey(Runtime.GetBytesForString (ComposeSortedKey(Key, recordIndex)), null); // read the indexed key byte[] kbuf1 = new byte[BufSize]; int klen1 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf1); Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1 ), ComposeSortedKey(Key, recordIndex)); klen1 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf1); Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1 ), ComposeSortedKey(Key, recordIndex)); klen1 = scanner.Entry().GetKeyLength(); scanner.Entry().GetKey(kbuf1); Assert.Equal(Runtime.GetStringForBytes(kbuf1, 0, klen1 ), ComposeSortedKey(Key, recordIndex)); scanner.Close(); reader.Close(); }
public virtual void TestSortedLongWritable() { Configuration conf = new Configuration(); Path path = new Path(Root, name); FileSystem fs = path.GetFileSystem(conf); FSDataOutputStream @out = fs.Create(path); try { TFile.Writer writer = new TFile.Writer(@out, BlockSize, "gz", jClassLongWritableComparator , conf); try { LongWritable key = new LongWritable(0); for (long i = 0; i < Nentry; ++i) { key.Set(Cube(i - Nentry / 2)); DataOutputStream dos = writer.PrepareAppendKey(-1); try { key.Write(dos); } finally { dos.Close(); } dos = writer.PrepareAppendValue(-1); try { dos.Write(Runtime.GetBytesForString(BuildValue(i))); } finally { dos.Close(); } } } finally { writer.Close(); } } finally { @out.Close(); } FSDataInputStream @in = fs.Open(path); try { TFile.Reader reader = new TFile.Reader(@in, fs.GetFileStatus(path).GetLen(), conf ); try { TFile.Reader.Scanner scanner = reader.CreateScanner(); long i = 0; BytesWritable value = new BytesWritable(); for (; !scanner.AtEnd(); scanner.Advance()) { scanner.Entry().GetValue(value); Assert.Equal(BuildValue(i), Runtime.GetStringForBytes( value.GetBytes(), 0, value.GetLength())); ++i; } } finally { reader.Close(); } } finally { @in.Close(); } }