/// <exception cref="System.IO.IOException"/> public override void DoTest(BlockReaderLocal reader, byte[] original) { byte[] buf = new byte[TestLength]; try { reader.ReadFully(buf, 0, 10); AssertArrayRegionsEqual(original, 0, buf, 0, 10); reader.ReadFully(buf, 10, 100); AssertArrayRegionsEqual(original, 10, buf, 10, 100); reader.ReadFully(buf, 110, 700); AssertArrayRegionsEqual(original, 110, buf, 110, 700); reader.Skip(1); // skip from offset 810 to offset 811 reader.ReadFully(buf, 811, 5); AssertArrayRegionsEqual(original, 811, buf, 811, 5); reader.ReadFully(buf, 816, 900); if (usingChecksums) { // We should detect the corruption when using a checksum file. NUnit.Framework.Assert.Fail("did not detect corruption"); } } catch (ChecksumException) { if (!usingChecksums) { NUnit.Framework.Assert.Fail("didn't expect to get ChecksumException: not " + "using checksums." ); } } }
/// <exception cref="System.IO.IOException"/> public override void DoTest(BlockReaderLocal reader, byte[] original) { byte[] emptyArr = new byte[0]; NUnit.Framework.Assert.AreEqual(0, reader.Read(emptyArr, 0, 0)); ByteBuffer emptyBuf = ByteBuffer.Wrap(emptyArr); NUnit.Framework.Assert.AreEqual(0, reader.Read(emptyBuf)); reader.Skip(1); NUnit.Framework.Assert.AreEqual(0, reader.Read(emptyArr, 0, 0)); NUnit.Framework.Assert.AreEqual(0, reader.Read(emptyBuf)); reader.Skip(TestBlockReaderLocal.BlockReaderLocalTest.TestLength - 1); NUnit.Framework.Assert.AreEqual(-1, reader.Read(emptyArr, 0, 0)); NUnit.Framework.Assert.AreEqual(-1, reader.Read(emptyBuf)); }
/// <exception cref="System.IO.IOException"/> public override void DoTest(BlockReaderLocal reader, byte[] original) { ByteBuffer buf = ByteBuffer.Wrap(new byte[TestLength]); ReadFully(reader, buf, 0, 10); AssertArrayRegionsEqual(original, 0, ((byte[])buf.Array()), 0, 10); ReadFully(reader, buf, 10, 100); AssertArrayRegionsEqual(original, 10, ((byte[])buf.Array()), 10, 100); ReadFully(reader, buf, 110, 700); AssertArrayRegionsEqual(original, 110, ((byte[])buf.Array()), 110, 700); reader.Skip(1); // skip from offset 810 to offset 811 ReadFully(reader, buf, 811, 5); AssertArrayRegionsEqual(original, 811, ((byte[])buf.Array()), 811, 5); }
/// <exception cref="System.IO.IOException"/> public override void DoTest(BlockReaderLocal reader, byte[] original) { byte[] buf = new byte[TestLength]; reader.ReadFully(buf, 0, 512); AssertArrayRegionsEqual(original, 0, buf, 0, 512); reader.ReadFully(buf, 512, 512); AssertArrayRegionsEqual(original, 512, buf, 512, 512); reader.ReadFully(buf, 1024, 513); AssertArrayRegionsEqual(original, 1024, buf, 1024, 513); reader.ReadFully(buf, 1537, 514); AssertArrayRegionsEqual(original, 1537, buf, 1537, 514); // Readahead is always at least the size of one chunk in this test. NUnit.Framework.Assert.IsTrue(reader.GetMaxReadaheadLength() >= TestBlockReaderLocal.BlockReaderLocalTest .BytesPerChecksum); }
/// <exception cref="System.IO.IOException"/> public override void DoTest(BlockReaderLocal reader, byte[] original) { byte[] buf = new byte[TestLength]; reader.ReadFully(buf, 0, 10); AssertArrayRegionsEqual(original, 0, buf, 0, 10); reader.ReadFully(buf, 10, 100); AssertArrayRegionsEqual(original, 10, buf, 10, 100); reader.ReadFully(buf, 110, 700); AssertArrayRegionsEqual(original, 110, buf, 110, 700); reader.ReadFully(buf, 810, 1); // from offset 810 to offset 811 reader.ReadFully(buf, 811, 5); AssertArrayRegionsEqual(original, 811, buf, 811, 5); reader.ReadFully(buf, 816, 900); // skip from offset 816 to offset 1716 reader.ReadFully(buf, 1716, 5); AssertArrayRegionsEqual(original, 1716, buf, 1716, 5); }
/// <exception cref="System.IO.IOException"/> public override void DoTest(BlockReaderLocal reader, byte[] original) { NUnit.Framework.Assert.IsTrue(!reader.GetVerifyChecksum()); ByteBuffer buf = ByteBuffer.Wrap(new byte[TestLength]); reader.Skip(1); ReadFully(reader, buf, 1, 9); AssertArrayRegionsEqual(original, 1, ((byte[])buf.Array()), 1, 9); ReadFully(reader, buf, 10, 100); AssertArrayRegionsEqual(original, 10, ((byte[])buf.Array()), 10, 100); reader.ForceAnchorable(); ReadFully(reader, buf, 110, 700); AssertArrayRegionsEqual(original, 110, ((byte[])buf.Array()), 110, 700); reader.ForceUnanchorable(); reader.Skip(1); // skip from offset 810 to offset 811 ReadFully(reader, buf, 811, 5); AssertArrayRegionsEqual(original, 811, ((byte[])buf.Array()), 811, 5); }
/// <summary>Similar to IOUtils#readFully().</summary> /// <remarks>Similar to IOUtils#readFully(). Reads bytes in a loop.</remarks> /// <param name="reader">The BlockReaderLocal to read bytes from</param> /// <param name="buf">The ByteBuffer to read into</param> /// <param name="off">The offset in the buffer to read into</param> /// <param name="len">The number of bytes to read.</param> /// <exception cref="System.IO.IOException">If it could not read the requested number of bytes /// </exception> private static void ReadFully(BlockReaderLocal reader, ByteBuffer buf, int off, int len) { int amt = len; while (amt > 0) { buf.Limit(off + len); buf.Position(off); long ret = reader.Read(buf); if (ret < 0) { throw new EOFException("Premature EOF from BlockReaderLocal " + "after reading " + (len - amt) + " byte(s)."); } amt -= ret; off += ret; } }
/// <exception cref="System.IO.IOException"/> public override void DoTest(BlockReaderLocal reader, byte[] original) { byte[] buf = new byte[TestLength]; if (usingChecksums) { try { reader.ReadFully(buf, 0, 10); NUnit.Framework.Assert.Fail("did not detect corruption"); } catch (IOException) { } } else { // expected reader.ReadFully(buf, 0, 10); } }
/// <exception cref="System.IO.IOException"/> public override void DoTest(BlockReaderLocal reader, byte[] original) { ByteBuffer buf = ByteBuffer.AllocateDirect(TestLength); ReadFully(reader, buf, 0, 5120); buf.Flip(); AssertArrayRegionsEqual(original, 0, DFSTestUtil.AsArray(buf), 0, 5120); reader.Skip(1537); ReadFully(reader, buf, 0, 1); buf.Flip(); AssertArrayRegionsEqual(original, 6657, DFSTestUtil.AsArray(buf), 0, 1); reader.ForceAnchorable(); ReadFully(reader, buf, 0, 5120); buf.Flip(); AssertArrayRegionsEqual(original, 6658, DFSTestUtil.AsArray(buf), 0, 5120); reader.ForceUnanchorable(); ReadFully(reader, buf, 0, 513); buf.Flip(); AssertArrayRegionsEqual(original, 11778, DFSTestUtil.AsArray(buf), 0, 513); reader.Skip(3); ReadFully(reader, buf, 0, 50); buf.Flip(); AssertArrayRegionsEqual(original, 12294, DFSTestUtil.AsArray(buf), 0, 50); }
// default: no-op /// <exception cref="System.IO.IOException"/> public virtual void DoTest(BlockReaderLocal reader, byte[] original) { }
/// <exception cref="System.IO.IOException"/> public virtual void RunBlockReaderLocalTest(TestBlockReaderLocal.BlockReaderLocalTest test, bool checksum, long readahead) { Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null )); MiniDFSCluster cluster = null; HdfsConfiguration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, !checksum ); conf.SetLong(DFSConfigKeys.DfsBytesPerChecksumKey, TestBlockReaderLocal.BlockReaderLocalTest .BytesPerChecksum); conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C"); conf.SetLong(DFSConfigKeys.DfsClientCacheReadahead, readahead); test.SetConfiguration(conf); FileInputStream dataIn = null; FileInputStream metaIn = null; Path TestPath = new Path("/a"); long RandomSeed = 4567L; BlockReaderLocal blockReaderLocal = null; FSDataInputStream fsIn = null; byte[] original = new byte[TestBlockReaderLocal.BlockReaderLocalTest.TestLength]; FileSystem fs = null; ShortCircuitShm shm = null; RandomAccessFile raf = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestBlockReaderLocal.BlockReaderLocalTest.TestLength , (short)1, RandomSeed); try { DFSTestUtil.WaitReplication(fs, TestPath, (short)1); } catch (Exception e) { NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: " + e); } catch (TimeoutException e) { NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: " + e); } fsIn = fs.Open(TestPath); IOUtils.ReadFully(fsIn, original, 0, TestBlockReaderLocal.BlockReaderLocalTest.TestLength ); fsIn.Close(); fsIn = null; ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestPath); FilePath dataFile = cluster.GetBlockFile(0, block); FilePath metaFile = cluster.GetBlockMetadataFile(0, block); ShortCircuitCache shortCircuitCache = ClientContext.GetFromConf(conf).GetShortCircuitCache (); cluster.Shutdown(); cluster = null; test.Setup(dataFile, checksum); FileInputStream[] streams = new FileInputStream[] { new FileInputStream(dataFile) , new FileInputStream(metaFile) }; dataIn = streams[0]; metaIn = streams[1]; ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId ()); raf = new RandomAccessFile(new FilePath(sockDir.GetDir().GetAbsolutePath(), UUID. RandomUUID().ToString()), "rw"); raf.SetLength(8192); FileInputStream shmStream = new FileInputStream(raf.GetFD()); shm = new ShortCircuitShm(ShortCircuitShm.ShmId.CreateRandom(), shmStream); ShortCircuitReplica replica = new ShortCircuitReplica(key, dataIn, metaIn, shortCircuitCache , Time.Now(), shm.AllocAndRegisterSlot(ExtendedBlockId.FromExtendedBlock(block)) ); blockReaderLocal = new BlockReaderLocal.Builder(new DFSClient.Conf(conf)).SetFilename (TestPath.GetName()).SetBlock(block).SetShortCircuitReplica(replica).SetCachingStrategy (new CachingStrategy(false, readahead)).SetVerifyChecksum(checksum).Build(); dataIn = null; metaIn = null; test.DoTest(blockReaderLocal, original); // BlockReaderLocal should not alter the file position. NUnit.Framework.Assert.AreEqual(0, streams[0].GetChannel().Position()); NUnit.Framework.Assert.AreEqual(0, streams[1].GetChannel().Position()); } finally { if (fsIn != null) { fsIn.Close(); } if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } if (dataIn != null) { dataIn.Close(); } if (metaIn != null) { metaIn.Close(); } if (blockReaderLocal != null) { blockReaderLocal.Close(); } if (shm != null) { shm.Free(); } if (raf != null) { raf.Close(); } } }