/// <exception cref="System.IO.IOException"/>
protected internal override int ReadChunk(long pos, byte[] buf, int offset, int len
, byte[] checksum)
{
bool eof = false;
if (NeedChecksum())
{
System.Diagnostics.Debug.Assert(checksum != null);
// we have a checksum buffer
System.Diagnostics.Debug.Assert(checksum.Length % ChecksumSize == 0);
// it is sane length
System.Diagnostics.Debug.Assert(len >= bytesPerSum);
// we must read at least one chunk
int checksumsToRead = Math.Min(len / bytesPerSum, checksum.Length / ChecksumSize);
// number of checksums based on len to read
// size of checksum buffer
long checksumPos = GetChecksumFilePos(pos);
if (checksumPos != sums.GetPos())
{
sums.Seek(checksumPos);
}
int sumLenRead = sums.Read(checksum, 0, ChecksumSize * checksumsToRead);
if (sumLenRead >= 0 && sumLenRead % ChecksumSize != 0)
{
throw new EOFException("Checksum file not a length multiple of checksum size " +
"in " + file + " at " + pos + " checksumpos: " + checksumPos + " sumLenread: " +
sumLenRead);
}
if (sumLenRead <= 0)
{
// we're at the end of the file
eof = true;
}
else
{
// Adjust amount of data to read based on how many checksum chunks we read
len = Math.Min(len, bytesPerSum * (sumLenRead / ChecksumSize));
}
}
if (pos != datas.GetPos())
{
datas.Seek(pos);
}
int nread = ReadFully(datas, buf, offset, len);
if (eof && nread > 0)
{
throw new ChecksumException("Checksum error: " + file + " at " + pos, pos);
}
return(nread);
}
/// <exception cref="System.IO.IOException"/>
internal static string ReadFile(FileSystem fs, Path name, int buflen)
{
byte[] b = new byte[buflen];
int offset = 0;
FSDataInputStream @in = fs.Open(name);
for (int remaining; (remaining = b.Length - offset) > 0 && (n = @in.Read(b, offset
, remaining)) != -1; offset += n)
{
}
Assert.Equal(offset, Math.Min(b.Length, @in.GetPos()));
@in.Close();
string s = Runtime.GetStringForBytes(b, 0, offset);
return(s);
}
/// <exception cref="System.IO.IOException"/>
public virtual long Tell()
{
return(stream.GetPos());
}
public virtual void Test2GBMmapLimit()
{
Assume.AssumeTrue(BlockReaderTestUtil.ShouldTestLargeFiles());
HdfsConfiguration conf = InitZeroCopyTest();
long TestFileLength = 2469605888L;
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "NULL");
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, TestFileLength);
MiniDFSCluster cluster = null;
Path TestPath = new Path("/a");
string Context = "test2GBMmapLimit";
conf.Set(DFSConfigKeys.DfsClientContext, Context);
FSDataInputStream fsIn = null;
FSDataInputStream fsIn2 = null;
ByteBuffer buf1 = null;
ByteBuffer buf2 = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, unchecked ((int)(0xB
)));
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
fsIn = fs.Open(TestPath);
buf1 = fsIn.Read(null, 1, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(1, buf1.Remaining());
fsIn.ReleaseBuffer(buf1);
buf1 = null;
fsIn.Seek(2147483640L);
buf1 = fsIn.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(7, buf1.Remaining());
NUnit.Framework.Assert.AreEqual(int.MaxValue, buf1.Limit());
fsIn.ReleaseBuffer(buf1);
buf1 = null;
NUnit.Framework.Assert.AreEqual(2147483647L, fsIn.GetPos());
try
{
buf1 = fsIn.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.Fail("expected UnsupportedOperationException");
}
catch (NotSupportedException)
{
}
// expected; can't read past 2GB boundary.
fsIn.Close();
fsIn = null;
// Now create another file with normal-sized blocks, and verify we
// can read past 2GB
Path TestPath2 = new Path("/b");
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 268435456L);
DFSTestUtil.CreateFile(fs, TestPath2, 1024 * 1024, TestFileLength, 268435456L, (short
)1, unchecked ((int)(0xA)));
fsIn2 = fs.Open(TestPath2);
fsIn2.Seek(2147483640L);
buf2 = fsIn2.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(8, buf2.Remaining());
NUnit.Framework.Assert.AreEqual(2147483648L, fsIn2.GetPos());
fsIn2.ReleaseBuffer(buf2);
buf2 = null;
buf2 = fsIn2.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(1024, buf2.Remaining());
NUnit.Framework.Assert.AreEqual(2147484672L, fsIn2.GetPos());
fsIn2.ReleaseBuffer(buf2);
buf2 = null;
}
finally
{
if (buf1 != null)
{
fsIn.ReleaseBuffer(buf1);
}
if (buf2 != null)
{
fsIn2.ReleaseBuffer(buf2);
}
IOUtils.Cleanup(null, fsIn, fsIn2);
if (cluster != null)
{
cluster.Shutdown();
}
}
}