public virtual void TestDisableCache()
{
HdfsConfiguration confWithoutCache = new HdfsConfiguration();
// Configure a new instance with no peer caching, ensure that it doesn't
// cache anything
confWithoutCache.SetInt(DFSConfigKeys.DfsClientSocketCacheCapacityKey, 0);
BlockReaderTestUtil util = new BlockReaderTestUtil(1, confWithoutCache);
Path testFile = new Path("/testConnCache.dat");
util.WriteFile(testFile, FileSize / 1024);
FileSystem fsWithoutCache = FileSystem.NewInstance(util.GetConf());
try
{
DFSTestUtil.ReadFile(fsWithoutCache, testFile);
NUnit.Framework.Assert.AreEqual(0, ((DistributedFileSystem)fsWithoutCache).dfs.GetClientContext
().GetPeerCache().Size());
}
finally
{
fsWithoutCache.Close();
util.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public static void SetupCluster(int replicationFactor, HdfsConfiguration conf)
{
util = new BlockReaderTestUtil(replicationFactor, conf);
dfsClient = util.GetDFSClient();
long seed = Time.Now();
Log.Info("Random seed: " + seed);
rand = new Random(seed);
}
public static void SetupCluster()
{
int ReplicationFactor = 1;
util = new BlockReaderTestUtil(ReplicationFactor);
util.WriteFile(TestFile, FileSizeK);
IList <LocatedBlock> blkList = util.GetFileBlocks(TestFile, FileSizeK);
testBlock = blkList[0];
}
public void Run()
{
try
{
IList <LocatedBlock> locatedBlocks = cluster.GetNameNode().GetRpcServer().GetBlockLocations
(TestFile, 0, TestFileLen).GetLocatedBlocks();
LocatedBlock lblock = locatedBlocks[0];
BlockReader blockReader = null;
try
{
blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen);
NUnit.Framework.Assert.Fail("expected getBlockReader to fail the first time.");
}
catch (Exception t)
{
NUnit.Framework.Assert.IsTrue("expected to see 'TCP reads were disabled " + "for testing' in exception "
+ t, t.Message.Contains("TCP reads were disabled for testing"));
}
finally
{
if (blockReader != null)
{
blockReader.Close();
}
}
gotFailureLatch.CountDown();
shouldRetryLatch.Await();
try
{
blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen);
}
catch (Exception t)
{
TestBlockReaderFactory.Log.Error("error trying to retrieve a block reader " + "the second time."
, t);
throw;
}
finally
{
if (blockReader != null)
{
blockReader.Close();
}
}
}
catch (Exception t)
{
TestBlockReaderFactory.Log.Error("getBlockReader failure", t);
testFailed.Set(true);
}
}
/// <summary>
/// Test the case where we have a failure to complete a short circuit read
/// that occurs, and then later on, we have a success.
/// </summary>
/// <remarks>
/// Test the case where we have a failure to complete a short circuit read
/// that occurs, and then later on, we have a success.
/// Any thread waiting on a cache load should receive the failure (if it
/// occurs); however, the failure result should not be cached. We want
/// to be able to retry later and succeed.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestShortCircuitCacheTemporaryFailure()
{
BlockReaderTestUtil.EnableBlockReaderFactoryTracing();
AtomicBoolean replicaCreationShouldFail = new AtomicBoolean(true);
AtomicBoolean testFailed = new AtomicBoolean(false);
DFSInputStream.tcpReadsDisabledForTesting = true;
BlockReaderFactory.createShortCircuitReplicaInfoCallback = new _ShortCircuitReplicaCreator_215
(replicaCreationShouldFail);
// Insert a short delay to increase the chance that one client
// thread waits for the other client thread's failure via
// a condition variable.
TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
Configuration conf = CreateShortCircuitConf("testShortCircuitCacheTemporaryFailure"
, sockDir);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
string TestFile = "/test_file";
int TestFileLen = 4000;
int NumThreads = 2;
int Seed = unchecked ((int)(0xFADED));
CountDownLatch gotFailureLatch = new CountDownLatch(NumThreads);
CountDownLatch shouldRetryLatch = new CountDownLatch(1);
DFSTestUtil.CreateFile(dfs, new Path(TestFile), TestFileLen, (short)1, Seed);
Runnable readerRunnable = new _Runnable_243(cluster, TestFile, TestFileLen, gotFailureLatch
, shouldRetryLatch, testFailed);
// First time should fail.
// first block
// keep findbugs happy
// Second time should succeed.
Sharpen.Thread[] threads = new Sharpen.Thread[NumThreads];
for (int i = 0; i < NumThreads; i++)
{
threads[i] = new Sharpen.Thread(readerRunnable);
threads[i].Start();
}
gotFailureLatch.Await();
replicaCreationShouldFail.Set(false);
shouldRetryLatch.CountDown();
for (int i_1 = 0; i_1 < NumThreads; i_1++)
{
Uninterruptibles.JoinUninterruptibly(threads[i_1]);
}
cluster.Shutdown();
sockDir.Close();
NUnit.Framework.Assert.IsFalse(testFailed.Get());
}
public void Run()
{
try
{
while (true)
{
BlockReader blockReader = null;
try
{
blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen);
sem.Release();
try
{
blockReader.ReadAll(buf, 0, TestFileLen);
}
finally
{
sem.AcquireUninterruptibly();
}
}
catch (ClosedByInterruptException e)
{
TestBlockReaderFactory.Log.Info("got the expected ClosedByInterruptException", e);
sem.Release();
break;
}
finally
{
if (blockReader != null)
{
blockReader.Close();
}
}
TestBlockReaderFactory.Log.Info("read another " + TestFileLen + " bytes.");
}
}
catch (Exception t)
{
TestBlockReaderFactory.Log.Error("getBlockReader failure", t);
testFailed.Set(true);
sem.Release();
}
}
public virtual void TestReadFromOneDN()
{
HdfsConfiguration configuration = new HdfsConfiguration();
// One of the goals of this test is to verify that we don't open more
// than one socket. So use a different client context, so that we
// get our own socket cache, rather than sharing with the other test
// instances. Also use a really long socket timeout so that nothing
// gets closed before we get around to checking the cache size at the end.
string contextName = "testReadFromOneDNContext";
configuration.Set(DFSConfigKeys.DfsClientContext, contextName);
configuration.SetLong(DFSConfigKeys.DfsClientSocketTimeoutKey, 100000000L);
BlockReaderTestUtil util = new BlockReaderTestUtil(1, configuration);
Path testFile = new Path("/testConnCache.dat");
byte[] authenticData = util.WriteFile(testFile, FileSize / 1024);
DFSClient client = new DFSClient(new IPEndPoint("localhost", util.GetCluster().GetNameNodePort
()), util.GetConf());
ClientContext cacheContext = ClientContext.Get(contextName, client.GetConf());
DFSInputStream @in = client.Open(testFile.ToString());
Log.Info("opened " + testFile.ToString());
byte[] dataBuf = new byte[BlockSize];
// Initial read
Pread(@in, 0, dataBuf, 0, dataBuf.Length, authenticData);
// Read again and verify that the socket is the same
Pread(@in, FileSize - dataBuf.Length, dataBuf, 0, dataBuf.Length, authenticData);
Pread(@in, 1024, dataBuf, 0, dataBuf.Length, authenticData);
// No seek; just read
Pread(@in, -1, dataBuf, 0, dataBuf.Length, authenticData);
Pread(@in, 64, dataBuf, 0, dataBuf.Length / 2, authenticData);
@in.Close();
client.Close();
NUnit.Framework.Assert.AreEqual(1, ClientContext.GetFromConf(configuration).GetPeerCache
().Size());
}
/// <summary>
/// When an InterruptedException is sent to a thread calling
/// FileChannel#read, the FileChannel is immediately closed and the
/// thread gets an exception.
/// </summary>
/// <remarks>
/// When an InterruptedException is sent to a thread calling
/// FileChannel#read, the FileChannel is immediately closed and the
/// thread gets an exception. This effectively means that we might have
/// someone asynchronously calling close() on the file descriptors we use
/// in BlockReaderLocal. So when unreferencing a ShortCircuitReplica in
/// ShortCircuitCache#unref, we should check if the FileChannel objects
/// are still open. If not, we should purge the replica to avoid giving
/// it out to any future readers.
/// This is a regression test for HDFS-6227: Short circuit read failed
/// due to ClosedChannelException.
/// Note that you may still get ClosedChannelException errors if two threads
/// are reading from the same replica and an InterruptedException is delivered
/// to one of them.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestPurgingClosedReplicas()
{
BlockReaderTestUtil.EnableBlockReaderFactoryTracing();
AtomicInteger replicasCreated = new AtomicInteger(0);
AtomicBoolean testFailed = new AtomicBoolean(false);
DFSInputStream.tcpReadsDisabledForTesting = true;
BlockReaderFactory.createShortCircuitReplicaInfoCallback = new _ShortCircuitReplicaCreator_443
(replicasCreated);
TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
Configuration conf = CreateShortCircuitConf("testPurgingClosedReplicas", sockDir);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
string TestFile = "/test_file";
int TestFileLen = 4095;
int Seed = unchecked ((int)(0xFADE0));
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI(0
), conf);
DFSTestUtil.CreateFile(fs, new Path(TestFile), TestFileLen, (short)1, Seed);
Semaphore sem = Sharpen.Extensions.CreateSemaphore(0);
IList <LocatedBlock> locatedBlocks = cluster.GetNameNode().GetRpcServer().GetBlockLocations
(TestFile, 0, TestFileLen).GetLocatedBlocks();
LocatedBlock lblock = locatedBlocks[0];
// first block
byte[] buf = new byte[TestFileLen];
Runnable readerRunnable = new _Runnable_471(cluster, lblock, TestFileLen, sem, buf
, testFailed);
Sharpen.Thread thread = new Sharpen.Thread(readerRunnable);
thread.Start();
// While the thread is reading, send it interrupts.
// These should trigger a ClosedChannelException.
while (thread.IsAlive())
{
sem.AcquireUninterruptibly();
thread.Interrupt();
sem.Release();
}
NUnit.Framework.Assert.IsFalse(testFailed.Get());
// We should be able to read from the file without
// getting a ClosedChannelException.
BlockReader blockReader = null;
try
{
blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen);
blockReader.ReadFully(buf, 0, TestFileLen);
}
finally
{
if (blockReader != null)
{
blockReader.Close();
}
}
byte[] expected = DFSTestUtil.CalculateFileContentsFromSeed(Seed, TestFileLen);
NUnit.Framework.Assert.IsTrue(Arrays.Equals(buf, expected));
// Another ShortCircuitReplica object should have been created.
NUnit.Framework.Assert.AreEqual(2, replicasCreated.Get());
dfs.Close();
cluster.Shutdown();
sockDir.Close();
}
