/// <exception cref="System.IO.IOException"/> protected internal override int ReadChunk(long pos, byte[] buf, int offset, int len , byte[] checksum) { bool eof = false; if (NeedChecksum()) { System.Diagnostics.Debug.Assert(checksum != null); // we have a checksum buffer System.Diagnostics.Debug.Assert(checksum.Length % ChecksumSize == 0); // it is sane length System.Diagnostics.Debug.Assert(len >= bytesPerSum); // we must read at least one chunk int checksumsToRead = Math.Min(len / bytesPerSum, checksum.Length / ChecksumSize); // number of checksums based on len to read // size of checksum buffer long checksumPos = GetChecksumFilePos(pos); if (checksumPos != sums.GetPos()) { sums.Seek(checksumPos); } int sumLenRead = sums.Read(checksum, 0, ChecksumSize * checksumsToRead); if (sumLenRead >= 0 && sumLenRead % ChecksumSize != 0) { throw new EOFException("Checksum file not a length multiple of checksum size " + "in " + file + " at " + pos + " checksumpos: " + checksumPos + " sumLenread: " + sumLenRead); } if (sumLenRead <= 0) { // we're at the end of the file eof = true; } else { // Adjust amount of data to read based on how many checksum chunks we read len = Math.Min(len, bytesPerSum * (sumLenRead / ChecksumSize)); } } if (pos != datas.GetPos()) { datas.Seek(pos); } int nread = ReadFully(datas, buf, offset, len); if (eof && nread > 0) { throw new ChecksumException("Checksum error: " + file + " at " + pos, pos); } return(nread); }
/// <exception cref="System.IO.IOException"/> internal static string ReadFile(FileSystem fs, Path name, int buflen) { byte[] b = new byte[buflen]; int offset = 0; FSDataInputStream @in = fs.Open(name); for (int remaining; (remaining = b.Length - offset) > 0 && (n = @in.Read(b, offset , remaining)) != -1; offset += n) { } Assert.Equal(offset, Math.Min(b.Length, @in.GetPos())); @in.Close(); string s = Runtime.GetStringForBytes(b, 0, offset); return(s); }
/// <exception cref="System.IO.IOException"/> public virtual long Tell() { return(stream.GetPos()); }
public virtual void Test2GBMmapLimit() { Assume.AssumeTrue(BlockReaderTestUtil.ShouldTestLargeFiles()); HdfsConfiguration conf = InitZeroCopyTest(); long TestFileLength = 2469605888L; conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "NULL"); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, TestFileLength); MiniDFSCluster cluster = null; Path TestPath = new Path("/a"); string Context = "test2GBMmapLimit"; conf.Set(DFSConfigKeys.DfsClientContext, Context); FSDataInputStream fsIn = null; FSDataInputStream fsIn2 = null; ByteBuffer buf1 = null; ByteBuffer buf2 = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); DistributedFileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, unchecked ((int)(0xB ))); DFSTestUtil.WaitReplication(fs, TestPath, (short)1); fsIn = fs.Open(TestPath); buf1 = fsIn.Read(null, 1, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.AreEqual(1, buf1.Remaining()); fsIn.ReleaseBuffer(buf1); buf1 = null; fsIn.Seek(2147483640L); buf1 = fsIn.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.AreEqual(7, buf1.Remaining()); NUnit.Framework.Assert.AreEqual(int.MaxValue, buf1.Limit()); fsIn.ReleaseBuffer(buf1); buf1 = null; NUnit.Framework.Assert.AreEqual(2147483647L, fsIn.GetPos()); try { buf1 = fsIn.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.Fail("expected UnsupportedOperationException"); } catch (NotSupportedException) { } // expected; can't read past 2GB boundary. fsIn.Close(); fsIn = null; // Now create another file with normal-sized blocks, and verify we // can read past 2GB Path TestPath2 = new Path("/b"); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 268435456L); DFSTestUtil.CreateFile(fs, TestPath2, 1024 * 1024, TestFileLength, 268435456L, (short )1, unchecked ((int)(0xA))); fsIn2 = fs.Open(TestPath2); fsIn2.Seek(2147483640L); buf2 = fsIn2.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.AreEqual(8, buf2.Remaining()); NUnit.Framework.Assert.AreEqual(2147483648L, fsIn2.GetPos()); fsIn2.ReleaseBuffer(buf2); buf2 = null; buf2 = fsIn2.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.AreEqual(1024, buf2.Remaining()); NUnit.Framework.Assert.AreEqual(2147484672L, fsIn2.GetPos()); fsIn2.ReleaseBuffer(buf2); buf2 = null; } finally { if (buf1 != null) { fsIn.ReleaseBuffer(buf1); } if (buf2 != null) { fsIn2.ReleaseBuffer(buf2); } IOUtils.Cleanup(null, fsIn, fsIn2); if (cluster != null) { cluster.Shutdown(); } } }