/// <exception cref="System.IO.IOException"/>
public override void DoTest(BlockReaderLocal reader, byte[] original)
{
byte[] buf = new byte[TestLength];
try
{
reader.ReadFully(buf, 0, 10);
AssertArrayRegionsEqual(original, 0, buf, 0, 10);
reader.ReadFully(buf, 10, 100);
AssertArrayRegionsEqual(original, 10, buf, 10, 100);
reader.ReadFully(buf, 110, 700);
AssertArrayRegionsEqual(original, 110, buf, 110, 700);
reader.Skip(1);
// skip from offset 810 to offset 811
reader.ReadFully(buf, 811, 5);
AssertArrayRegionsEqual(original, 811, buf, 811, 5);
reader.ReadFully(buf, 816, 900);
if (usingChecksums)
{
// We should detect the corruption when using a checksum file.
NUnit.Framework.Assert.Fail("did not detect corruption");
}
}
catch (ChecksumException)
{
if (!usingChecksums)
{
NUnit.Framework.Assert.Fail("didn't expect to get ChecksumException: not " + "using checksums."
);
}
}
}
/// <exception cref="System.IO.IOException"/>
public override void DoTest(BlockReaderLocal reader, byte[] original)
{
byte[] emptyArr = new byte[0];
NUnit.Framework.Assert.AreEqual(0, reader.Read(emptyArr, 0, 0));
ByteBuffer emptyBuf = ByteBuffer.Wrap(emptyArr);
NUnit.Framework.Assert.AreEqual(0, reader.Read(emptyBuf));
reader.Skip(1);
NUnit.Framework.Assert.AreEqual(0, reader.Read(emptyArr, 0, 0));
NUnit.Framework.Assert.AreEqual(0, reader.Read(emptyBuf));
reader.Skip(TestBlockReaderLocal.BlockReaderLocalTest.TestLength - 1);
NUnit.Framework.Assert.AreEqual(-1, reader.Read(emptyArr, 0, 0));
NUnit.Framework.Assert.AreEqual(-1, reader.Read(emptyBuf));
}
/// <exception cref="System.IO.IOException"/>
public override void DoTest(BlockReaderLocal reader, byte[] original)
{
ByteBuffer buf = ByteBuffer.Wrap(new byte[TestLength]);
ReadFully(reader, buf, 0, 10);
AssertArrayRegionsEqual(original, 0, ((byte[])buf.Array()), 0, 10);
ReadFully(reader, buf, 10, 100);
AssertArrayRegionsEqual(original, 10, ((byte[])buf.Array()), 10, 100);
ReadFully(reader, buf, 110, 700);
AssertArrayRegionsEqual(original, 110, ((byte[])buf.Array()), 110, 700);
reader.Skip(1);
// skip from offset 810 to offset 811
ReadFully(reader, buf, 811, 5);
AssertArrayRegionsEqual(original, 811, ((byte[])buf.Array()), 811, 5);
}
/// <exception cref="System.IO.IOException"/>
public override void DoTest(BlockReaderLocal reader, byte[] original)
{
byte[] buf = new byte[TestLength];
reader.ReadFully(buf, 0, 512);
AssertArrayRegionsEqual(original, 0, buf, 0, 512);
reader.ReadFully(buf, 512, 512);
AssertArrayRegionsEqual(original, 512, buf, 512, 512);
reader.ReadFully(buf, 1024, 513);
AssertArrayRegionsEqual(original, 1024, buf, 1024, 513);
reader.ReadFully(buf, 1537, 514);
AssertArrayRegionsEqual(original, 1537, buf, 1537, 514);
// Readahead is always at least the size of one chunk in this test.
NUnit.Framework.Assert.IsTrue(reader.GetMaxReadaheadLength() >= TestBlockReaderLocal.BlockReaderLocalTest
.BytesPerChecksum);
}
/// <exception cref="System.IO.IOException"/>
public override void DoTest(BlockReaderLocal reader, byte[] original)
{
byte[] buf = new byte[TestLength];
reader.ReadFully(buf, 0, 10);
AssertArrayRegionsEqual(original, 0, buf, 0, 10);
reader.ReadFully(buf, 10, 100);
AssertArrayRegionsEqual(original, 10, buf, 10, 100);
reader.ReadFully(buf, 110, 700);
AssertArrayRegionsEqual(original, 110, buf, 110, 700);
reader.ReadFully(buf, 810, 1);
// from offset 810 to offset 811
reader.ReadFully(buf, 811, 5);
AssertArrayRegionsEqual(original, 811, buf, 811, 5);
reader.ReadFully(buf, 816, 900);
// skip from offset 816 to offset 1716
reader.ReadFully(buf, 1716, 5);
AssertArrayRegionsEqual(original, 1716, buf, 1716, 5);
}
/// <exception cref="System.IO.IOException"/>
public override void DoTest(BlockReaderLocal reader, byte[] original)
{
NUnit.Framework.Assert.IsTrue(!reader.GetVerifyChecksum());
ByteBuffer buf = ByteBuffer.Wrap(new byte[TestLength]);
reader.Skip(1);
ReadFully(reader, buf, 1, 9);
AssertArrayRegionsEqual(original, 1, ((byte[])buf.Array()), 1, 9);
ReadFully(reader, buf, 10, 100);
AssertArrayRegionsEqual(original, 10, ((byte[])buf.Array()), 10, 100);
reader.ForceAnchorable();
ReadFully(reader, buf, 110, 700);
AssertArrayRegionsEqual(original, 110, ((byte[])buf.Array()), 110, 700);
reader.ForceUnanchorable();
reader.Skip(1);
// skip from offset 810 to offset 811
ReadFully(reader, buf, 811, 5);
AssertArrayRegionsEqual(original, 811, ((byte[])buf.Array()), 811, 5);
}
/// <summary>Similar to IOUtils#readFully().</summary>
/// <remarks>Similar to IOUtils#readFully(). Reads bytes in a loop.</remarks>
/// <param name="reader">The BlockReaderLocal to read bytes from</param>
/// <param name="buf">The ByteBuffer to read into</param>
/// <param name="off">The offset in the buffer to read into</param>
/// <param name="len">The number of bytes to read.</param>
/// <exception cref="System.IO.IOException">If it could not read the requested number of bytes
/// </exception>
private static void ReadFully(BlockReaderLocal reader, ByteBuffer buf, int off, int
len)
{
int amt = len;
while (amt > 0)
{
buf.Limit(off + len);
buf.Position(off);
long ret = reader.Read(buf);
if (ret < 0)
{
throw new EOFException("Premature EOF from BlockReaderLocal " + "after reading "
+ (len - amt) + " byte(s).");
}
amt -= ret;
off += ret;
}
}
/// <exception cref="System.IO.IOException"/>
public override void DoTest(BlockReaderLocal reader, byte[] original)
{
byte[] buf = new byte[TestLength];
if (usingChecksums)
{
try
{
reader.ReadFully(buf, 0, 10);
NUnit.Framework.Assert.Fail("did not detect corruption");
}
catch (IOException)
{
}
}
else
{
// expected
reader.ReadFully(buf, 0, 10);
}
}
/// <exception cref="System.IO.IOException"/>
public override void DoTest(BlockReaderLocal reader, byte[] original)
{
ByteBuffer buf = ByteBuffer.AllocateDirect(TestLength);
ReadFully(reader, buf, 0, 5120);
buf.Flip();
AssertArrayRegionsEqual(original, 0, DFSTestUtil.AsArray(buf), 0, 5120);
reader.Skip(1537);
ReadFully(reader, buf, 0, 1);
buf.Flip();
AssertArrayRegionsEqual(original, 6657, DFSTestUtil.AsArray(buf), 0, 1);
reader.ForceAnchorable();
ReadFully(reader, buf, 0, 5120);
buf.Flip();
AssertArrayRegionsEqual(original, 6658, DFSTestUtil.AsArray(buf), 0, 5120);
reader.ForceUnanchorable();
ReadFully(reader, buf, 0, 513);
buf.Flip();
AssertArrayRegionsEqual(original, 11778, DFSTestUtil.AsArray(buf), 0, 513);
reader.Skip(3);
ReadFully(reader, buf, 0, 50);
buf.Flip();
AssertArrayRegionsEqual(original, 12294, DFSTestUtil.AsArray(buf), 0, 50);
}
// default: no-op
/// <exception cref="System.IO.IOException"/>
public virtual void DoTest(BlockReaderLocal reader, byte[] original)
{
}
/// <exception cref="System.IO.IOException"/>
public virtual void RunBlockReaderLocalTest(TestBlockReaderLocal.BlockReaderLocalTest
test, bool checksum, long readahead)
{
Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null
));
MiniDFSCluster cluster = null;
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, !checksum
);
conf.SetLong(DFSConfigKeys.DfsBytesPerChecksumKey, TestBlockReaderLocal.BlockReaderLocalTest
.BytesPerChecksum);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C");
conf.SetLong(DFSConfigKeys.DfsClientCacheReadahead, readahead);
test.SetConfiguration(conf);
FileInputStream dataIn = null;
FileInputStream metaIn = null;
Path TestPath = new Path("/a");
long RandomSeed = 4567L;
BlockReaderLocal blockReaderLocal = null;
FSDataInputStream fsIn = null;
byte[] original = new byte[TestBlockReaderLocal.BlockReaderLocalTest.TestLength];
FileSystem fs = null;
ShortCircuitShm shm = null;
RandomAccessFile raf = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
, (short)1, RandomSeed);
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
fsIn = fs.Open(TestPath);
IOUtils.ReadFully(fsIn, original, 0, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
);
fsIn.Close();
fsIn = null;
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestPath);
FilePath dataFile = cluster.GetBlockFile(0, block);
FilePath metaFile = cluster.GetBlockMetadataFile(0, block);
ShortCircuitCache shortCircuitCache = ClientContext.GetFromConf(conf).GetShortCircuitCache
();
cluster.Shutdown();
cluster = null;
test.Setup(dataFile, checksum);
FileInputStream[] streams = new FileInputStream[] { new FileInputStream(dataFile)
, new FileInputStream(metaFile) };
dataIn = streams[0];
metaIn = streams[1];
ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId
());
raf = new RandomAccessFile(new FilePath(sockDir.GetDir().GetAbsolutePath(), UUID.
RandomUUID().ToString()), "rw");
raf.SetLength(8192);
FileInputStream shmStream = new FileInputStream(raf.GetFD());
shm = new ShortCircuitShm(ShortCircuitShm.ShmId.CreateRandom(), shmStream);
ShortCircuitReplica replica = new ShortCircuitReplica(key, dataIn, metaIn, shortCircuitCache
, Time.Now(), shm.AllocAndRegisterSlot(ExtendedBlockId.FromExtendedBlock(block))
);
blockReaderLocal = new BlockReaderLocal.Builder(new DFSClient.Conf(conf)).SetFilename
(TestPath.GetName()).SetBlock(block).SetShortCircuitReplica(replica).SetCachingStrategy
(new CachingStrategy(false, readahead)).SetVerifyChecksum(checksum).Build();
dataIn = null;
metaIn = null;
test.DoTest(blockReaderLocal, original);
// BlockReaderLocal should not alter the file position.
NUnit.Framework.Assert.AreEqual(0, streams[0].GetChannel().Position());
NUnit.Framework.Assert.AreEqual(0, streams[1].GetChannel().Position());
}
finally
{
if (fsIn != null)
{
fsIn.Close();
}
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
if (dataIn != null)
{
dataIn.Close();
}
if (metaIn != null)
{
metaIn.Close();
}
if (blockReaderLocal != null)
{
blockReaderLocal.Close();
}
if (shm != null)
{
shm.Free();
}
if (raf != null)
{
raf.Close();
}
}
}
