public virtual void TestGetPos()
{
Path testFile = new Path("/testfile+1");
// Write a test file.
FSDataOutputStream @out = hdfs.Create(testFile, true);
@out.WriteBytes("0123456789");
@out.Close();
FSDataInputStream @in = hftpFs.Open(testFile);
// Test read().
for (int i = 0; i < 5; ++i)
{
NUnit.Framework.Assert.AreEqual(i, @in.GetPos());
@in.Read();
}
// Test read(b, off, len).
NUnit.Framework.Assert.AreEqual(5, @in.GetPos());
byte[] buffer = new byte[10];
NUnit.Framework.Assert.AreEqual(2, @in.Read(buffer, 0, 2));
NUnit.Framework.Assert.AreEqual(7, @in.GetPos());
// Test read(b).
int bytesRead = @in.Read(buffer);
NUnit.Framework.Assert.AreEqual(7 + bytesRead, @in.GetPos());
// Test EOF.
for (int i_1 = 0; i_1 < 100; ++i_1)
{
@in.Read();
}
NUnit.Framework.Assert.AreEqual(10, @in.GetPos());
@in.Close();
}
public virtual void TestNegativeSeek()
{
instream = GetFileSystem().Open(smallSeekFile);
Assert.Equal(0, instream.GetPos());
try
{
instream.Seek(-1);
long p = instream.GetPos();
Log.Warn("Seek to -1 returned a position of " + p);
int result = instream.Read();
NUnit.Framework.Assert.Fail("expected an exception, got data " + result + " at a position of "
+ p);
}
catch (EOFException e)
{
//bad seek -expected
HandleExpectedException(e);
}
catch (IOException e)
{
//bad seek -expected, but not as preferred as an EOFException
HandleRelaxedException("a negative seek", "EOFException", e);
}
Assert.Equal(0, instream.GetPos());
}
public virtual void TestSeekBigFile()
{
Describe("Seek round a large file and verify the bytes are what is expected");
Path testSeekFile = Path("bigseekfile.txt");
byte[] block = ContractTestUtils.Dataset(65536, 0, 255);
ContractTestUtils.CreateFile(GetFileSystem(), testSeekFile, false, block);
instream = GetFileSystem().Open(testSeekFile);
Assert.Equal(0, instream.GetPos());
//expect that seek to 0 works
instream.Seek(0);
int result = instream.Read();
Assert.Equal(0, result);
Assert.Equal(1, instream.Read());
Assert.Equal(2, instream.Read());
//do seek 32KB ahead
instream.Seek(32768);
Assert.Equal("@32768", block[32768], unchecked ((byte)instream.
Read()));
instream.Seek(40000);
Assert.Equal("@40000", block[40000], unchecked ((byte)instream.
Read()));
instream.Seek(8191);
Assert.Equal("@8191", block[8191], unchecked ((byte)instream.Read
()));
instream.Seek(0);
Assert.Equal("@0", 0, unchecked ((byte)instream.Read()));
}
/// <exception cref="System.IO.IOException"/>
private long DumpFromOffset(PathData item, long offset)
{
long fileSize = item.RefreshStatus().GetLen();
if (offset > fileSize)
{
return(fileSize);
}
// treat a negative offset as relative to end of the file, floor of 0
if (offset < 0)
{
offset = Math.Max(fileSize + offset, 0);
}
FSDataInputStream @in = item.fs.Open(item.path);
try
{
@in.Seek(offset);
// use conf so the system configured io block size is used
IOUtils.CopyBytes(@in, System.Console.Out, GetConf(), false);
offset = @in.GetPos();
}
finally
{
@in.Close();
}
return(offset);
}
/// <exception cref="System.Exception"/>
private void ReadTestFile(string testFileName)
{
Path filePath = new Path(testFileName);
FSDataInputStream istream = dfs.Open(filePath, 10240);
ByteBuffer buf = ByteBuffer.Allocate(10240);
int count = 0;
try
{
while (istream.Read(buf) > 0)
{
count += 1;
buf.Clear();
istream.Seek(istream.GetPos() + 5);
}
}
catch (IOException)
{
}
finally
{
// Ignore this it's probably a seek after eof.
istream.Close();
}
}
public virtual void TestSeekFile()
{
Describe("basic seek operations");
instream = GetFileSystem().Open(smallSeekFile);
Assert.Equal(0, instream.GetPos());
//expect that seek to 0 works
instream.Seek(0);
int result = instream.Read();
Assert.Equal(0, result);
Assert.Equal(1, instream.Read());
Assert.Equal(2, instream.GetPos());
Assert.Equal(2, instream.Read());
Assert.Equal(3, instream.GetPos());
instream.Seek(128);
Assert.Equal(128, instream.GetPos());
Assert.Equal(128, instream.Read());
instream.Seek(63);
Assert.Equal(63, instream.Read());
}
public virtual void TestBlockReadZeroByteFile()
{
Describe("do a block read on a 0 byte file");
instream = GetFileSystem().Open(zeroByteFile);
Assert.Equal(0, instream.GetPos());
//expect that seek to 0 works
byte[] buffer = new byte[1];
int result = instream.Read(buffer, 0, 1);
AssertMinusOne("block read zero byte file", result);
}
public virtual void TestSeekAndReadPastEndOfFile()
{
Describe("verify that reading past the last bytes in the file returns -1");
instream = GetFileSystem().Open(smallSeekFile);
Assert.Equal(0, instream.GetPos());
//expect that seek to 0 works
//go just before the end
instream.Seek(TestFileLen - 2);
Assert.True("Premature EOF", instream.Read() != -1);
Assert.True("Premature EOF", instream.Read() != -1);
AssertMinusOne("read past end of file", instream.Read());
}
public virtual void TestOpenReadZeroByteFile()
{
Describe("create & read a 0 byte file");
Path path = Path("zero.txt");
ContractTestUtils.Touch(GetFileSystem(), path);
instream = GetFileSystem().Open(path);
Assert.Equal(0, instream.GetPos());
//expect initial read to fail
int result = instream.Read();
AssertMinusOne("initial byte read", result);
}
/// <exception cref="System.IO.IOException"/>
protected internal override int ReadChunk(long pos, byte[] buf, int offset, int len
, byte[] checksum)
{
bool eof = false;
if (NeedChecksum())
{
System.Diagnostics.Debug.Assert(checksum != null);
// we have a checksum buffer
System.Diagnostics.Debug.Assert(checksum.Length % ChecksumSize == 0);
// it is sane length
System.Diagnostics.Debug.Assert(len >= bytesPerSum);
// we must read at least one chunk
int checksumsToRead = Math.Min(len / bytesPerSum, checksum.Length / ChecksumSize);
// number of checksums based on len to read
// size of checksum buffer
long checksumPos = GetChecksumFilePos(pos);
if (checksumPos != sums.GetPos())
{
sums.Seek(checksumPos);
}
int sumLenRead = sums.Read(checksum, 0, ChecksumSize * checksumsToRead);
if (sumLenRead >= 0 && sumLenRead % ChecksumSize != 0)
{
throw new ChecksumException("Checksum file not a length multiple of checksum size "
+ "in " + file + " at " + pos + " checksumpos: " + checksumPos + " sumLenread: "
+ sumLenRead, pos);
}
if (sumLenRead <= 0)
{
// we're at the end of the file
eof = true;
}
else
{
// Adjust amount of data to read based on how many checksum chunks we read
len = Math.Min(len, bytesPerSum * (sumLenRead / ChecksumSize));
}
}
if (pos != datas.GetPos())
{
datas.Seek(pos);
}
int nread = ReadFully(datas, buf, offset, len);
if (eof && nread > 0)
{
throw new ChecksumException("Checksum error: " + file + " at " + pos, pos);
}
return(nread);
}
public virtual void TestPositionedBulkReadDoesntChangePosition()
{
Describe("verify that a positioned read does not change the getPos() value");
Path testSeekFile = Path("bigseekfile.txt");
byte[] block = ContractTestUtils.Dataset(65536, 0, 255);
ContractTestUtils.CreateFile(GetFileSystem(), testSeekFile, false, block);
instream = GetFileSystem().Open(testSeekFile);
instream.Seek(39999);
Assert.True(-1 != instream.Read());
Assert.Equal(40000, instream.GetPos());
byte[] readBuffer = new byte[256];
instream.Read(128, readBuffer, 0, readBuffer.Length);
//have gone back
Assert.Equal(40000, instream.GetPos());
//content is the same too
Assert.Equal("@40000", block[40000], unchecked ((byte)instream.
Read()));
//now verify the picked up data
for (int i = 0; i < 256; i++)
{
Assert.Equal("@" + i, block[i + 128], readBuffer[i]);
}
}
public virtual void TestSeekZeroByteFile()
{
Describe("seek and read a 0 byte file");
instream = GetFileSystem().Open(zeroByteFile);
Assert.Equal(0, instream.GetPos());
//expect initial read to fai;
int result = instream.Read();
AssertMinusOne("initial byte read", result);
byte[] buffer = new byte[1];
//expect that seek to 0 works
instream.Seek(0);
//reread, expect same exception
result = instream.Read();
AssertMinusOne("post-seek byte read", result);
result = instream.Read(buffer, 0, 1);
AssertMinusOne("post-seek buffer read", result);
}
public virtual void TestRamDiskEvictionWithShortCircuitReadHandle()
{
StartUpCluster(ReplFactor, new StorageType[] { StorageType.RamDisk, StorageType.Default }, (6 * BlockSize - 1), true);
// 5 replica + delta, SCR.
string MethodName = GenericTestUtils.GetMethodName();
Path path1 = new Path("/" + MethodName + ".01.dat");
Path path2 = new Path("/" + MethodName + ".02.dat");
int Seed = unchecked ((int)(0xFADED));
MakeRandomTestFile(path1, BlockSize, true, Seed);
EnsureFileReplicasOnStorageType(path1, StorageType.RamDisk);
// Sleep for a short time to allow the lazy writer thread to do its job.
// However the block replica should not be evicted from RAM_DISK yet.
Sharpen.Thread.Sleep(3 * LazyWriterIntervalSec * 1000);
// No eviction should happen as the free ratio is below the threshold
FSDataInputStream fis = fs.Open(path1);
try
{
// Keep and open read handle to path1 while creating path2
byte[] buf = new byte[BufferLength];
fis.Read(0, buf, 0, BufferLength);
// Create the 2nd file that will trigger RAM_DISK eviction.
MakeTestFile(path2, BlockSize * 2, true);
EnsureFileReplicasOnStorageType(path2, StorageType.RamDisk);
// Ensure path1 is still readable from the open SCR handle.
fis.Read(fis.GetPos(), buf, 0, BufferLength);
HdfsDataInputStream dfsis = (HdfsDataInputStream)fis;
NUnit.Framework.Assert.AreEqual(2 * BufferLength, dfsis.GetReadStatistics().GetTotalBytesRead
());
NUnit.Framework.Assert.AreEqual(2 * BufferLength, dfsis.GetReadStatistics().GetTotalShortCircuitBytesRead
());
}
finally
{
IOUtils.CloseQuietly(fis);
}
// After the open handle is closed, path1 should be evicted to DISK.
TriggerBlockReport();
EnsureFileReplicasOnStorageType(path1, StorageType.Default);
}
/// <summary>
/// Test (expected to throw IOE) for <code>FSDataInpuStream#seek</code>
/// when the position argument is larger than the file size.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestSeekPastFileSize()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
Path seekFile = new Path("seekboundaries.dat");
DFSTestUtil.CreateFile(fs, seekFile, Onemb, fs.GetDefaultReplication(seekFile), seed
);
FSDataInputStream stream = fs.Open(seekFile);
// Perform "safe seek" (expected to pass)
stream.Seek(65536);
NUnit.Framework.Assert.AreEqual(65536, stream.GetPos());
// expect IOE for this call
stream.Seek(Onemb + Onemb + Onemb);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
/// <summary>read chunks into buffer repeatedly until total of VisibleLen byte are read.
/// </summary>
/// <remarks>
/// read chunks into buffer repeatedly until total of VisibleLen byte are read.
/// Return total number of bytes read
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private long ReadUntilEnd(FSDataInputStream @in, byte[] buffer, long size, string
fname, long pos, long visibleLen, bool positionReadOption)
{
if (pos >= visibleLen || visibleLen <= 0)
{
return(0);
}
int chunkNumber = 0;
long totalByteRead = 0;
long currentPosition = pos;
int byteRead = 0;
long byteLeftToRead = visibleLen - pos;
int byteToReadThisRound = 0;
if (!positionReadOption)
{
@in.Seek(pos);
currentPosition = @in.GetPos();
}
if (verboseOption)
{
Log.Info("reader begin: position: " + pos + " ; currentOffset = " + currentPosition
+ " ; bufferSize =" + buffer.Length + " ; Filename = " + fname);
}
try
{
while (byteLeftToRead > 0 && currentPosition < visibleLen)
{
byteToReadThisRound = (int)(byteLeftToRead >= buffer.Length ? buffer.Length : byteLeftToRead
);
if (positionReadOption)
{
byteRead = @in.Read(currentPosition, buffer, 0, byteToReadThisRound);
}
else
{
byteRead = @in.Read(buffer, 0, byteToReadThisRound);
}
if (byteRead <= 0)
{
break;
}
chunkNumber++;
totalByteRead += byteRead;
currentPosition += byteRead;
byteLeftToRead -= byteRead;
if (verboseOption)
{
Log.Info("reader: Number of byte read: " + byteRead + " ; totalByteRead = " + totalByteRead
+ " ; currentPosition=" + currentPosition + " ; chunkNumber =" + chunkNumber +
"; File name = " + fname);
}
}
}
catch (IOException e)
{
throw new IOException("#### Exception caught in readUntilEnd: reader currentOffset = "
+ currentPosition + " ; totalByteRead =" + totalByteRead + " ; latest byteRead = "
+ byteRead + "; visibleLen= " + visibleLen + " ; bufferLen = " + buffer.Length
+ " ; Filename = " + fname, e);
}
if (verboseOption)
{
Log.Info("reader end: position: " + pos + " ; currentOffset = " + currentPosition
+ " ; totalByteRead =" + totalByteRead + " ; Filename = " + fname);
}
return(totalByteRead);
}
/* test read and getPos */
/// <exception cref="System.Exception"/>
private void CheckReadAndGetPos()
{
actual = new byte[FileSize];
// test reads that do not cross checksum boundary
stm.Seek(0);
int offset;
for (offset = 0; offset < BlockSize + BytesPerSum; offset += BytesPerSum)
{
NUnit.Framework.Assert.AreEqual(stm.GetPos(), offset);
stm.ReadFully(actual, offset, BytesPerSum);
}
stm.ReadFully(actual, offset, FileSize - BlockSize - BytesPerSum);
NUnit.Framework.Assert.AreEqual(stm.GetPos(), FileSize);
CheckAndEraseData(actual, 0, expected, "Read Sanity Test");
// test reads that cross checksum boundary
stm.Seek(0L);
NUnit.Framework.Assert.AreEqual(stm.GetPos(), 0L);
stm.ReadFully(actual, 0, HalfChunkSize);
NUnit.Framework.Assert.AreEqual(stm.GetPos(), HalfChunkSize);
stm.ReadFully(actual, HalfChunkSize, BlockSize - HalfChunkSize);
NUnit.Framework.Assert.AreEqual(stm.GetPos(), BlockSize);
stm.ReadFully(actual, BlockSize, BytesPerSum + HalfChunkSize);
NUnit.Framework.Assert.AreEqual(stm.GetPos(), BlockSize + BytesPerSum + HalfChunkSize
);
stm.ReadFully(actual, 2 * BlockSize - HalfChunkSize, FileSize - (2 * BlockSize -
HalfChunkSize));
NUnit.Framework.Assert.AreEqual(stm.GetPos(), FileSize);
CheckAndEraseData(actual, 0, expected, "Read Sanity Test");
// test read that cross block boundary
stm.Seek(0L);
stm.ReadFully(actual, 0, BytesPerSum + HalfChunkSize);
NUnit.Framework.Assert.AreEqual(stm.GetPos(), BytesPerSum + HalfChunkSize);
stm.ReadFully(actual, BytesPerSum + HalfChunkSize, BytesPerSum);
NUnit.Framework.Assert.AreEqual(stm.GetPos(), BlockSize + HalfChunkSize);
stm.ReadFully(actual, BlockSize + HalfChunkSize, FileSize - BlockSize - HalfChunkSize
);
NUnit.Framework.Assert.AreEqual(stm.GetPos(), FileSize);
CheckAndEraseData(actual, 0, expected, "Read Sanity Test");
}
public virtual void TestSeekReadClosedFile()
{
bool supportsSeekOnClosedFiles = IsSupported(SupportsSeekOnClosedFile);
instream = GetFileSystem().Open(smallSeekFile);
GetLog().Debug("Stream is of type " + instream.GetType().GetCanonicalName());
instream.Close();
try
{
instream.Seek(0);
if (!supportsSeekOnClosedFiles)
{
NUnit.Framework.Assert.Fail("seek succeeded on a closed stream");
}
}
catch (IOException)
{
}
//expected a closed file
try
{
int data = instream.Available();
NUnit.Framework.Assert.Fail("read() succeeded on a closed stream, got " + data);
}
catch (IOException)
{
}
//expected a closed file
try
{
int data = instream.Read();
NUnit.Framework.Assert.Fail("read() succeeded on a closed stream, got " + data);
}
catch (IOException)
{
}
//expected a closed file
try
{
byte[] buffer = new byte[1];
int result = instream.Read(buffer, 0, 1);
NUnit.Framework.Assert.Fail("read(buffer, 0, 1) succeeded on a closed stream, got "
+ result);
}
catch (IOException)
{
}
//expected a closed file
//what position does a closed file have?
try
{
long offset = instream.GetPos();
}
catch (IOException)
{
}
// its valid to raise error here; but the test is applied to make
// sure there's no other exception like an NPE.
//and close again
instream.Close();
}