public virtual void RemoveShm(ShortCircuitShm shm)
{
lock (this)
{
if (Log.IsTraceEnabled())
{
Log.Debug("removing shm " + shm);
}
// Stop tracking the shmId.
ShortCircuitRegistry.RegisteredShm removedShm = Sharpen.Collections.Remove(segments
, shm.GetShmId());
Preconditions.CheckState(removedShm == shm, "failed to remove " + shm.GetShmId());
// Stop tracking the slots.
for (IEnumerator <ShortCircuitShm.Slot> iter = shm.SlotIterator(); iter.HasNext();)
{
ShortCircuitShm.Slot slot = iter.Next();
bool removed = slots.Remove(slot.GetBlockId(), slot);
Preconditions.CheckState(removed);
slot.MakeInvalid();
}
// De-allocate the memory map and close the shared file.
shm.Free();
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void RunBlockReaderLocalTest(TestBlockReaderLocal.BlockReaderLocalTest
test, bool checksum, long readahead)
{
Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null
));
MiniDFSCluster cluster = null;
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, !checksum
);
conf.SetLong(DFSConfigKeys.DfsBytesPerChecksumKey, TestBlockReaderLocal.BlockReaderLocalTest
.BytesPerChecksum);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C");
conf.SetLong(DFSConfigKeys.DfsClientCacheReadahead, readahead);
test.SetConfiguration(conf);
FileInputStream dataIn = null;
FileInputStream metaIn = null;
Path TestPath = new Path("/a");
long RandomSeed = 4567L;
BlockReaderLocal blockReaderLocal = null;
FSDataInputStream fsIn = null;
byte[] original = new byte[TestBlockReaderLocal.BlockReaderLocalTest.TestLength];
FileSystem fs = null;
ShortCircuitShm shm = null;
RandomAccessFile raf = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
, (short)1, RandomSeed);
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
fsIn = fs.Open(TestPath);
IOUtils.ReadFully(fsIn, original, 0, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
);
fsIn.Close();
fsIn = null;
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestPath);
FilePath dataFile = cluster.GetBlockFile(0, block);
FilePath metaFile = cluster.GetBlockMetadataFile(0, block);
ShortCircuitCache shortCircuitCache = ClientContext.GetFromConf(conf).GetShortCircuitCache
();
cluster.Shutdown();
cluster = null;
test.Setup(dataFile, checksum);
FileInputStream[] streams = new FileInputStream[] { new FileInputStream(dataFile)
, new FileInputStream(metaFile) };
dataIn = streams[0];
metaIn = streams[1];
ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId
());
raf = new RandomAccessFile(new FilePath(sockDir.GetDir().GetAbsolutePath(), UUID.
RandomUUID().ToString()), "rw");
raf.SetLength(8192);
FileInputStream shmStream = new FileInputStream(raf.GetFD());
shm = new ShortCircuitShm(ShortCircuitShm.ShmId.CreateRandom(), shmStream);
ShortCircuitReplica replica = new ShortCircuitReplica(key, dataIn, metaIn, shortCircuitCache
, Time.Now(), shm.AllocAndRegisterSlot(ExtendedBlockId.FromExtendedBlock(block))
);
blockReaderLocal = new BlockReaderLocal.Builder(new DFSClient.Conf(conf)).SetFilename
(TestPath.GetName()).SetBlock(block).SetShortCircuitReplica(replica).SetCachingStrategy
(new CachingStrategy(false, readahead)).SetVerifyChecksum(checksum).Build();
dataIn = null;
metaIn = null;
test.DoTest(blockReaderLocal, original);
// BlockReaderLocal should not alter the file position.
NUnit.Framework.Assert.AreEqual(0, streams[0].GetChannel().Position());
NUnit.Framework.Assert.AreEqual(0, streams[1].GetChannel().Position());
}
finally
{
if (fsIn != null)
{
fsIn.Close();
}
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
if (dataIn != null)
{
dataIn.Close();
}
if (metaIn != null)
{
metaIn.Close();
}
if (blockReaderLocal != null)
{
blockReaderLocal.Close();
}
if (shm != null)
{
shm.Free();
}
if (raf != null)
{
raf.Close();
}
}
}