public virtual void TestSeekBigFile()
{
Describe("Seek round a large file and verify the bytes are what is expected");
Path testSeekFile = Path("bigseekfile.txt");
byte[] block = ContractTestUtils.Dataset(65536, 0, 255);
ContractTestUtils.CreateFile(GetFileSystem(), testSeekFile, false, block);
instream = GetFileSystem().Open(testSeekFile);
Assert.Equal(0, instream.GetPos());
//expect that seek to 0 works
instream.Seek(0);
int result = instream.Read();
Assert.Equal(0, result);
Assert.Equal(1, instream.Read());
Assert.Equal(2, instream.Read());
//do seek 32KB ahead
instream.Seek(32768);
Assert.Equal("@32768", block[32768], unchecked ((byte)instream.
Read()));
instream.Seek(40000);
Assert.Equal("@40000", block[40000], unchecked ((byte)instream.
Read()));
instream.Seek(8191);
Assert.Equal("@8191", block[8191], unchecked ((byte)instream.Read
()));
instream.Seek(0);
Assert.Equal("@0", 0, unchecked ((byte)instream.Read()));
}
/// <summary>Constructor</summary>
/// <param name="fin">FS input stream.</param>
/// <param name="fileLength">Length of the corresponding file</param>
/// <exception cref="System.IO.IOException"/>
public Reader(FSDataInputStream fin, long fileLength, Configuration conf)
{
this.@in = fin;
this.conf = conf;
// move the cursor to the beginning of the tail, containing: offset to the
// meta block index, version and magic
fin.Seek(fileLength - BCFile.Magic.Size() - Utils.Version.Size() - long.Size / byte
.Size);
long offsetIndexMeta = fin.ReadLong();
version = new Utils.Version(fin);
BCFile.Magic.ReadAndVerify(fin);
if (!version.CompatibleWith(BCFile.ApiVersion))
{
throw new RuntimeException("Incompatible BCFile fileBCFileVersion.");
}
// read meta index
fin.Seek(offsetIndexMeta);
metaIndex = new BCFile.MetaIndex(fin);
// read data:BCFile.index, the data block index
BCFile.Reader.BlockReader blockR = GetMetaBlock(BCFile.DataIndex.BlockName);
try
{
dataIndex = new BCFile.DataIndex(blockR);
}
finally
{
blockR.Close();
}
}
/*
* Read some data, skip a few bytes and read more. HADOOP-922.
*/
/// <exception cref="System.IO.IOException"/>
private void SmallReadSeek(FileSystem fileSys, Path name)
{
if (fileSys is ChecksumFileSystem)
{
fileSys = ((ChecksumFileSystem)fileSys).GetRawFileSystem();
}
// Make the buffer size small to trigger code for HADOOP-922
FSDataInputStream stmRaw = fileSys.Open(name, 1);
byte[] expected = new byte[Onemb];
Random rand = new Random(seed);
rand.NextBytes(expected);
// Issue a simple read first.
byte[] actual = new byte[128];
stmRaw.Seek(100000);
stmRaw.Read(actual, 0, actual.Length);
CheckAndEraseData(actual, 100000, expected, "First Small Read Test");
// now do a small seek of 4 bytes, within the same block.
int newpos1 = 100000 + 128 + 4;
stmRaw.Seek(newpos1);
stmRaw.Read(actual, 0, actual.Length);
CheckAndEraseData(actual, newpos1, expected, "Small Seek Bug 1");
// seek another 256 bytes this time
int newpos2 = newpos1 + 256;
stmRaw.Seek(newpos2);
stmRaw.Read(actual, 0, actual.Length);
CheckAndEraseData(actual, newpos2, expected, "Small Seek Bug 2");
// all done
stmRaw.Close();
}
public virtual void TestNegativeSeek()
{
instream = GetFileSystem().Open(smallSeekFile);
Assert.Equal(0, instream.GetPos());
try
{
instream.Seek(-1);
long p = instream.GetPos();
Log.Warn("Seek to -1 returned a position of " + p);
int result = instream.Read();
NUnit.Framework.Assert.Fail("expected an exception, got data " + result + " at a position of "
+ p);
}
catch (EOFException e)
{
//bad seek -expected
HandleExpectedException(e);
}
catch (IOException e)
{
//bad seek -expected, but not as preferred as an EOFException
HandleRelaxedException("a negative seek", "EOFException", e);
}
Assert.Equal(0, instream.GetPos());
}
/// <exception cref="System.IO.IOException"/>
private long DumpFromOffset(PathData item, long offset)
{
long fileSize = item.RefreshStatus().GetLen();
if (offset > fileSize)
{
return(fileSize);
}
// treat a negative offset as relative to end of the file, floor of 0
if (offset < 0)
{
offset = Math.Max(fileSize + offset, 0);
}
FSDataInputStream @in = item.fs.Open(item.path);
try
{
@in.Seek(offset);
// use conf so the system configured io block size is used
IOUtils.CopyBytes(@in, System.Console.Out, GetConf(), false);
offset = @in.GetPos();
}
finally
{
@in.Close();
}
return(offset);
}
public virtual void TestReadClosedStream()
{
Path testFile = new Path("/testfile+2");
FSDataOutputStream os = hdfs.Create(testFile, true);
os.WriteBytes("0123456789");
os.Close();
// ByteRangeInputStream delays opens until reads. Make sure it doesn't
// open a closed stream that has never been opened
FSDataInputStream @in = hftpFs.Open(testFile);
@in.Close();
CheckClosedStream(@in);
CheckClosedStream(@in.GetWrappedStream());
// force the stream to connect and then close it
@in = hftpFs.Open(testFile);
int ch = @in.Read();
NUnit.Framework.Assert.AreEqual('0', ch);
@in.Close();
CheckClosedStream(@in);
CheckClosedStream(@in.GetWrappedStream());
// make sure seeking doesn't automagically reopen the stream
@in.Seek(4);
CheckClosedStream(@in);
CheckClosedStream(@in.GetWrappedStream());
}
/// <exception cref="System.IO.IOException"/>
public CombineFileLineRecordReader(CombineFileSplit split, TaskAttemptContext context
, int index)
{
//offset of the chunk;
//end of the chunk;
// current pos
this.path = split.GetPath(index);
fs = this.path.GetFileSystem(context.GetConfiguration());
this.startOffset = split.GetOffset(index);
this.end = startOffset + split.GetLength(index);
bool skipFirstLine = false;
//open the file
fileIn = fs.Open(path);
if (startOffset != 0)
{
skipFirstLine = true;
--startOffset;
fileIn.Seek(startOffset);
}
reader = new LineReader(fileIn);
if (skipFirstLine)
{
// skip first line and re-establish "startOffset".
startOffset += reader.ReadLine(new Text(), 0, (int)Math.Min((long)int.MaxValue, end
- startOffset));
}
this.pos = startOffset;
}
/// <exception cref="System.IO.IOException"/>
public override int Read(byte[] b, int off, int len)
{
if ((off | len | (off + len) | (b.Length - (off + len))) < 0)
{
throw new IndexOutOfRangeException();
}
int n = (int)Math.Min(int.MaxValue, Math.Min(len, (end - pos)));
if (n == 0)
{
return(-1);
}
int ret = 0;
lock (@in)
{
@in.Seek(pos);
ret = @in.Read(b, off, n);
}
if (ret < 0)
{
end = pos;
return(-1);
}
pos += ret;
return(ret);
}
/// <summary>test seek</summary>
/// <exception cref="System.IO.IOException"/>
internal static void VerifySeek(FileSystem fs, Path p, long offset, long length,
byte[] buf, byte[] expected)
{
long remaining = length - offset;
long @checked = 0;
Log.Info("XXX SEEK: offset=" + offset + ", remaining=" + remaining);
TestWebHDFS.Ticker t = new TestWebHDFS.Ticker("SEEK", "offset=%d, remaining=%d",
offset, remaining);
FSDataInputStream @in = fs.Open(p, 64 << 10);
@in.Seek(offset);
for (; remaining > 0;)
{
t.Tick(@checked, "offset=%d, remaining=%d", offset, remaining);
int n = (int)Math.Min(remaining, buf.Length);
@in.ReadFully(buf, 0, n);
CheckData(offset, remaining, n, buf, expected);
offset += n;
remaining -= n;
@checked += n;
}
@in.Close();
t.End(@checked);
}
/// <exception cref="System.Exception"/>
private void ReadTestFile(string testFileName)
{
Path filePath = new Path(testFileName);
FSDataInputStream istream = dfs.Open(filePath, 10240);
ByteBuffer buf = ByteBuffer.Allocate(10240);
int count = 0;
try
{
while (istream.Read(buf) > 0)
{
count += 1;
buf.Clear();
istream.Seek(istream.GetPos() + 5);
}
}
catch (IOException)
{
}
finally
{
// Ignore this it's probably a seek after eof.
istream.Close();
}
}
/// <summary>
/// Verify that the read at a specific offset in a stream
/// matches that expected
/// </summary>
/// <param name="stm">stream</param>
/// <param name="fileContents">original file contents</param>
/// <param name="seekOff">seek offset</param>
/// <param name="toRead">number of bytes to read</param>
/// <exception cref="System.IO.IOException">IO problems</exception>
public static void VerifyRead(FSDataInputStream stm, byte[] fileContents, int seekOff
, int toRead)
{
byte[] @out = new byte[toRead];
stm.Seek(seekOff);
stm.ReadFully(@out);
byte[] expected = Arrays.CopyOfRange(fileContents, seekOff, seekOff + toRead);
CompareByteArrays(expected, @out, toRead);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSeek()
{
Path dir = new Path("/test/testSeek");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir));
{
//test zero file size
Path zero = new Path(dir, "zero");
fs.Create(zero).Close();
int count = 0;
FSDataInputStream @in = fs.Open(zero);
for (; @in.Read() != -1; count++)
{
}
@in.Close();
NUnit.Framework.Assert.AreEqual(0, count);
}
byte[] mydata = new byte[1 << 20];
new Random().NextBytes(mydata);
Path p = new Path(dir, "file");
FSDataOutputStream @out = fs.Create(p, false, 4096, (short)3, 1L << 17);
@out.Write(mydata, 0, mydata.Length);
@out.Close();
int one_third = mydata.Length / 3;
int two_third = one_third * 2;
{
//test seek
int offset = one_third;
int len = mydata.Length - offset;
byte[] buf = new byte[len];
FSDataInputStream @in = fs.Open(p);
@in.Seek(offset);
//read all remaining data
@in.ReadFully(buf);
@in.Close();
for (int i = 0; i < buf.Length; i++)
{
NUnit.Framework.Assert.AreEqual("Position " + i + ", offset=" + offset + ", length="
+ len, mydata[i + offset], buf[i]);
}
}
{
//test position read (read the data after the two_third location)
int offset = two_third;
int len = mydata.Length - offset;
byte[] buf = new byte[len];
FSDataInputStream @in = fs.Open(p);
@in.ReadFully(offset, buf);
@in.Close();
for (int i = 0; i < buf.Length; i++)
{
NUnit.Framework.Assert.AreEqual("Position " + i + ", offset=" + offset + ", length="
+ len, mydata[i + offset], buf[i]);
}
}
}
/// <exception cref="System.IO.IOException"/>
private void ReadAndCompare(FSDataInputStream @in, int position, int len)
{
byte[] b = new byte[len];
@in.Seek(position);
IOUtils.ReadFully(@in, b, 0, b.Length);
for (int i = 0; i < b.Length; i++)
{
NUnit.Framework.Assert.AreEqual(expected[position + i], b[i]);
}
}
public virtual void TestSeekFile()
{
Describe("basic seek operations");
instream = GetFileSystem().Open(smallSeekFile);
Assert.Equal(0, instream.GetPos());
//expect that seek to 0 works
instream.Seek(0);
int result = instream.Read();
Assert.Equal(0, result);
Assert.Equal(1, instream.Read());
Assert.Equal(2, instream.GetPos());
Assert.Equal(2, instream.Read());
Assert.Equal(3, instream.GetPos());
instream.Seek(128);
Assert.Equal(128, instream.GetPos());
Assert.Equal(128, instream.Read());
instream.Seek(63);
Assert.Equal(63, instream.Read());
}
public virtual void TestSeekAndReadPastEndOfFile()
{
Describe("verify that reading past the last bytes in the file returns -1");
instream = GetFileSystem().Open(smallSeekFile);
Assert.Equal(0, instream.GetPos());
//expect that seek to 0 works
//go just before the end
instream.Seek(TestFileLen - 2);
Assert.True("Premature EOF", instream.Read() != -1);
Assert.True("Premature EOF", instream.Read() != -1);
AssertMinusOne("read past end of file", instream.Read());
}
/// <exception cref="System.IO.IOException"/>
public override void Initialize(InputSplit genericSplit, TaskAttemptContext context
)
{
FileSplit split = (FileSplit)genericSplit;
Configuration job = context.GetConfiguration();
this.maxLineLength = job.GetInt(MaxLineLength, int.MaxValue);
start = split.GetStart();
end = start + split.GetLength();
Path file = split.GetPath();
// open the file and seek to the start of the split
FileSystem fs = file.GetFileSystem(job);
fileIn = fs.Open(file);
CompressionCodec codec = new CompressionCodecFactory(job).GetCodec(file);
if (null != codec)
{
isCompressedInput = true;
decompressor = CodecPool.GetDecompressor(codec);
if (codec is SplittableCompressionCodec)
{
SplitCompressionInputStream cIn = ((SplittableCompressionCodec)codec).CreateInputStream
(fileIn, decompressor, start, end, SplittableCompressionCodec.READ_MODE.Byblock);
@in = new CompressedSplitLineReader(cIn, job, this.recordDelimiterBytes);
start = cIn.GetAdjustedStart();
end = cIn.GetAdjustedEnd();
filePosition = cIn;
}
else
{
@in = new SplitLineReader(codec.CreateInputStream(fileIn, decompressor), job, this
.recordDelimiterBytes);
filePosition = fileIn;
}
}
else
{
fileIn.Seek(start);
@in = new UncompressedSplitLineReader(fileIn, job, this.recordDelimiterBytes, split
.GetLength());
filePosition = fileIn;
}
// If this is not the first split, we always throw away first record
// because we always (except the last split) read one extra line in
// next() method.
if (start != 0)
{
start += @in.ReadLine(new Text(), 0, MaxBytesToConsume(start));
}
this.pos = start;
}
/// <summary>
/// Retrieve the map output of a single map task
/// and send it to the merger.
/// </summary>
/// <exception cref="System.IO.IOException"/>
private bool CopyMapOutput(TaskAttemptID mapTaskId)
{
// Figure out where the map task stored its output.
Path mapOutputFileName = localMapFiles[mapTaskId].GetOutputFile();
Path indexFileName = mapOutputFileName.Suffix(".index");
// Read its index to determine the location of our split
// and its size.
SpillRecord sr = new SpillRecord(indexFileName, job);
IndexRecord ir = sr.GetIndex(reduce);
long compressedLength = ir.partLength;
long decompressedLength = ir.rawLength;
compressedLength -= CryptoUtils.CryptoPadding(job);
decompressedLength -= CryptoUtils.CryptoPadding(job);
// Get the location for the map output - either in-memory or on-disk
MapOutput <K, V> mapOutput = merger.Reserve(mapTaskId, decompressedLength, id);
// Check if we can shuffle *now* ...
if (mapOutput == null)
{
Log.Info("fetcher#" + id + " - MergeManager returned Status.WAIT ...");
return(false);
}
// Go!
Log.Info("localfetcher#" + id + " about to shuffle output of map " + mapOutput.GetMapId
() + " decomp: " + decompressedLength + " len: " + compressedLength + " to " + mapOutput
.GetDescription());
// now read the file, seek to the appropriate section, and send it.
FileSystem localFs = FileSystem.GetLocal(job).GetRaw();
FSDataInputStream inStream = localFs.Open(mapOutputFileName);
inStream = CryptoUtils.WrapIfNecessary(job, inStream);
try
{
inStream.Seek(ir.startOffset + CryptoUtils.CryptoPadding(job));
mapOutput.Shuffle(Localhost, inStream, compressedLength, decompressedLength, metrics
, reporter);
}
finally
{
try
{
inStream.Close();
}
catch (IOException ioe)
{
Log.Warn("IOException closing inputstream from map output: " + ioe.ToString());
}
}
scheduler.CopySucceeded(mapTaskId, Localhost, compressedLength, 0, 0, mapOutput);
return(true);
}
public virtual void TestSeek()
{
Path testFile = new Path("/testfile+1");
FSDataOutputStream @out = hdfs.Create(testFile, true);
@out.WriteBytes("0123456789");
@out.Close();
FSDataInputStream @in = hftpFs.Open(testFile);
@in.Seek(7);
NUnit.Framework.Assert.AreEqual('7', @in.Read());
@in.Close();
}
public virtual void TestSeekPastEndOfFileThenReseekAndRead()
{
Describe("do a seek past the EOF, then verify the stream recovers");
instream = GetFileSystem().Open(smallSeekFile);
//go just before the end. This may or may not fail; it may be delayed until the
//read
bool canSeekPastEOF = !GetContract().IsSupported(ContractOptions.RejectsSeekPastEof
, true);
try
{
instream.Seek(TestFileLen + 1);
//if this doesn't trigger, then read() is expected to fail
AssertMinusOne("read after seeking past EOF", instream.Read());
}
catch (EOFException e)
{
//This is an error iff the FS claims to be able to seek past the EOF
if (canSeekPastEOF)
{
//a failure wasn't expected
throw;
}
HandleExpectedException(e);
}
catch (IOException e)
{
//This is an error iff the FS claims to be able to seek past the EOF
if (canSeekPastEOF)
{
//a failure wasn't expected
throw;
}
HandleRelaxedException("a seek past the end of the file", "EOFException", e);
}
//now go back and try to read from a valid point in the file
instream.Seek(1);
Assert.True("Premature EOF", instream.Read() != -1);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void Initialize(InputSplit split, TaskAttemptContext context)
{
Path p = ((FileSplit)split).GetPath();
FileSystem fs = p.GetFileSystem(context.GetConfiguration());
@in = fs.Open(p);
long start = ((FileSplit)split).GetStart();
// find the offset to start at a record boundary
offset = (RecordLength - (start % RecordLength)) % RecordLength;
@in.Seek(start + offset);
length = ((FileSplit)split).GetLength();
}
/// <exception cref="System.IO.IOException"/>
protected internal override int ReadChunk(long pos, byte[] buf, int offset, int len
, byte[] checksum)
{
bool eof = false;
if (NeedChecksum())
{
System.Diagnostics.Debug.Assert(checksum != null);
// we have a checksum buffer
System.Diagnostics.Debug.Assert(checksum.Length % ChecksumSize == 0);
// it is sane length
System.Diagnostics.Debug.Assert(len >= bytesPerSum);
// we must read at least one chunk
int checksumsToRead = Math.Min(len / bytesPerSum, checksum.Length / ChecksumSize);
// number of checksums based on len to read
// size of checksum buffer
long checksumPos = GetChecksumFilePos(pos);
if (checksumPos != sums.GetPos())
{
sums.Seek(checksumPos);
}
int sumLenRead = sums.Read(checksum, 0, ChecksumSize * checksumsToRead);
if (sumLenRead >= 0 && sumLenRead % ChecksumSize != 0)
{
throw new ChecksumException("Checksum file not a length multiple of checksum size "
+ "in " + file + " at " + pos + " checksumpos: " + checksumPos + " sumLenread: "
+ sumLenRead, pos);
}
if (sumLenRead <= 0)
{
// we're at the end of the file
eof = true;
}
else
{
// Adjust amount of data to read based on how many checksum chunks we read
len = Math.Min(len, bytesPerSum * (sumLenRead / ChecksumSize));
}
}
if (pos != datas.GetPos())
{
datas.Seek(pos);
}
int nread = ReadFully(datas, buf, offset, len);
if (eof && nread > 0)
{
throw new ChecksumException("Checksum error: " + file + " at " + pos, pos);
}
return(nread);
}
/// <exception cref="System.IO.IOException"/>
public LineRecordReader(Configuration job, FileSplit split, byte[] recordDelimiter
)
{
this.maxLineLength = job.GetInt(LineRecordReader.MaxLineLength, int.MaxValue);
start = split.GetStart();
end = start + split.GetLength();
Path file = split.GetPath();
compressionCodecs = new CompressionCodecFactory(job);
codec = compressionCodecs.GetCodec(file);
// open the file and seek to the start of the split
FileSystem fs = file.GetFileSystem(job);
fileIn = fs.Open(file);
if (IsCompressedInput())
{
decompressor = CodecPool.GetDecompressor(codec);
if (codec is SplittableCompressionCodec)
{
SplitCompressionInputStream cIn = ((SplittableCompressionCodec)codec).CreateInputStream
(fileIn, decompressor, start, end, SplittableCompressionCodec.READ_MODE.Byblock);
@in = new CompressedSplitLineReader(cIn, job, recordDelimiter);
start = cIn.GetAdjustedStart();
end = cIn.GetAdjustedEnd();
filePosition = cIn;
}
else
{
// take pos from compressed stream
@in = new SplitLineReader(codec.CreateInputStream(fileIn, decompressor), job, recordDelimiter
);
filePosition = fileIn;
}
}
else
{
fileIn.Seek(start);
@in = new UncompressedSplitLineReader(fileIn, job, recordDelimiter, split.GetLength
());
filePosition = fileIn;
}
// If this is not the first split, we always throw away first record
// because we always (except the last split) read one extra line in
// next() method.
if (start != 0)
{
start += @in.ReadLine(new Text(), 0, MaxBytesToConsume(start));
}
this.pos = start;
}
/// <exception cref="System.IO.IOException"/>
private void TestEOF(MiniDFSCluster cluster, int fileLength)
{
FileSystem fs = cluster.GetFileSystem();
Path path = new Path("testEOF." + fileLength);
DFSTestUtil.CreateFile(fs, path, fileLength, (short)1, unchecked ((int)(0xBEEFBEEF
)));
FSDataInputStream fis = fs.Open(path);
ByteBuffer empty = ByteBuffer.Allocate(0);
// A read into an empty bytebuffer at the beginning of the file gives 0.
NUnit.Framework.Assert.AreEqual(0, fis.Read(empty));
fis.Seek(fileLength);
// A read into an empty bytebuffer at the end of the file gives -1.
NUnit.Framework.Assert.AreEqual(-1, fis.Read(empty));
if (fileLength > BlockSize)
{
fis.Seek(fileLength - BlockSize + 1);
ByteBuffer dbb = ByteBuffer.AllocateDirect(BlockSize);
NUnit.Framework.Assert.AreEqual(BlockSize - 1, fis.Read(dbb));
}
fis.Close();
}
// This is also called from the old FixedLengthRecordReader API implementation
/// <exception cref="System.IO.IOException"/>
public virtual void Initialize(Configuration job, long splitStart, long splitLength
, Path file)
{
start = splitStart;
end = start + splitLength;
long partialRecordLength = start % recordLength;
long numBytesToSkip = 0;
if (partialRecordLength != 0)
{
numBytesToSkip = recordLength - partialRecordLength;
}
// open the file and seek to the start of the split
FileSystem fs = file.GetFileSystem(job);
fileIn = fs.Open(file);
CompressionCodec codec = new CompressionCodecFactory(job).GetCodec(file);
if (null != codec)
{
isCompressedInput = true;
decompressor = CodecPool.GetDecompressor(codec);
CompressionInputStream cIn = codec.CreateInputStream(fileIn, decompressor);
filePosition = cIn;
inputStream = cIn;
numRecordsRemainingInSplit = long.MaxValue;
Log.Info("Compressed input; cannot compute number of records in the split");
}
else
{
fileIn.Seek(start);
filePosition = fileIn;
inputStream = fileIn;
long splitSize = end - start - numBytesToSkip;
numRecordsRemainingInSplit = (splitSize + recordLength - 1) / recordLength;
if (numRecordsRemainingInSplit < 0)
{
numRecordsRemainingInSplit = 0;
}
Log.Info("Expecting " + numRecordsRemainingInSplit + " records each with a length of "
+ recordLength + " bytes in the split with an effective size of " + splitSize +
" bytes");
}
if (numBytesToSkip != 0)
{
start += inputStream.Skip(numBytesToSkip);
}
this.pos = start;
}
/// <exception cref="System.IO.IOException"/>
internal virtual void Init(Counters.Counter readsCounter)
{
if (reader == null)
{
FSDataInputStream @in = fs.Open(file);
@in.Seek(segmentOffset);
@in = CryptoUtils.WrapIfNecessary(conf, @in);
reader = new IFile.Reader <K, V>(conf, @in, segmentLength - CryptoUtils.CryptoPadding
(conf), codec, readsCounter);
}
if (mapOutputsCounter != null)
{
mapOutputsCounter.Increment(1);
}
}
/// <summary>
/// Test (expected to throw IOE) for <code>FSDataInpuStream#seek</code>
/// when the position argument is larger than the file size.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestSeekPastFileSize()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
Path seekFile = new Path("seekboundaries.dat");
DFSTestUtil.CreateFile(fs, seekFile, Onemb, fs.GetDefaultReplication(seekFile), seed
);
FSDataInputStream stream = fs.Open(seekFile);
// Perform "safe seek" (expected to pass)
stream.Seek(65536);
NUnit.Framework.Assert.AreEqual(65536, stream.GetPos());
// expect IOE for this call
stream.Seek(Onemb + Onemb + Onemb);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
public virtual void TestSeekZeroByteFile()
{
Describe("seek and read a 0 byte file");
instream = GetFileSystem().Open(zeroByteFile);
Assert.Equal(0, instream.GetPos());
//expect initial read to fai;
int result = instream.Read();
AssertMinusOne("initial byte read", result);
byte[] buffer = new byte[1];
//expect that seek to 0 works
instream.Seek(0);
//reread, expect same exception
result = instream.Read();
AssertMinusOne("post-seek byte read", result);
result = instream.Read(buffer, 0, 1);
AssertMinusOne("post-seek buffer read", result);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSkipWithVerifyChecksum()
{
int size = blockSize;
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "/tmp/testSkipWithVerifyChecksum._PORT"
);
DomainSocket.DisableBindPathValidation();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Format(
true).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
// check that / exists
Path path = new Path("/");
NUnit.Framework.Assert.IsTrue("/ should be a directory", fs.GetFileStatus(path).IsDirectory
() == true);
byte[] fileData = AppendTestUtil.RandomBytes(seed, size * 3);
// create a new file in home directory. Do not close it.
Path file1 = new Path("filelocal.dat");
FSDataOutputStream stm = CreateFile(fs, file1, 1);
// write to file
stm.Write(fileData);
stm.Close();
// now test the skip function
FSDataInputStream instm = fs.Open(file1);
byte[] actual = new byte[fileData.Length];
// read something from the block first, otherwise BlockReaderLocal.skip()
// will not be invoked
int nread = instm.Read(actual, 0, 3);
long skipped = 2 * size + 3;
instm.Seek(skipped);
nread = instm.Read(actual, (int)(skipped + nread), 3);
instm.Close();
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestSeekAfterSetDropBehind()
{
// start a cluster
Log.Info("testSeekAfterSetDropBehind");
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
string TestPath = "/test";
int TestPathLen = MaxTestFileLen;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
CreateHdfsFile(fs, new Path(TestPath), TestPathLen, false);
// verify that we can seek after setDropBehind
FSDataInputStream fis = fs.Open(new Path(TestPath));
try
{
NUnit.Framework.Assert.IsTrue(fis.Read() != -1);
// create BlockReader
fis.SetDropBehind(false);
// clear BlockReader
fis.Seek(2);
}
finally
{
// seek
fis.Close();
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
private long TailFile(Path file, long startPos)
{
long numRead = 0;
FSDataInputStream inputStream = fileSystem.Open(file);
inputStream.Seek(startPos);
int len = 4 * 1024;
byte[] buf = new byte[len];
int read;
while ((read = inputStream.Read(buf)) > -1)
{
Log.Info(string.Format("read %d bytes", read));
if (!ValidateSequentialBytes(buf, (int)(startPos + numRead), read))
{
Log.Error(string.Format("invalid bytes: [%s]\n", Arrays.ToString(buf)));
throw new ChecksumException(string.Format("unable to validate bytes"), startPos);
}
numRead += read;
}
inputStream.Close();
return(numRead + startPos - 1);
}