/// <summary>Test that the client respects its keepalive timeout.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestClientResponsesKeepAliveTimeout()
{
Configuration clientConf = new Configuration(conf);
// Set a client socket cache expiry time much shorter than
// the datanode-side expiration time.
long ClientExpiryMs = 10L;
clientConf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, ClientExpiryMs
);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testClientResponsesKeepAliveTimeout"
);
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI()
, clientConf);
PeerCache peerCache = ClientContext.GetFromConf(clientConf).GetPeerCache();
DFSTestUtil.CreateFile(fs, TestFile, 1L, (short)1, 0L);
// Clients that write aren't currently re-used.
NUnit.Framework.Assert.AreEqual(0, peerCache.Size());
AssertXceiverCount(0);
// Reads the file, so we should get a
// cached socket, and should have an xceiver on the other side.
DFSTestUtil.ReadFile(fs, TestFile);
NUnit.Framework.Assert.AreEqual(1, peerCache.Size());
AssertXceiverCount(1);
// Sleep for a bit longer than the client keepalive timeout.
Sharpen.Thread.Sleep(ClientExpiryMs + 50);
// Taking out a peer which is expired should give a null.
Peer peer = peerCache.Get(dn.GetDatanodeId(), false);
NUnit.Framework.Assert.IsTrue(peer == null);
// The socket cache is now empty.
NUnit.Framework.Assert.AreEqual(0, peerCache.Size());
}
public virtual void TestMultiplePeersWithSameKey()
{
int Capacity = 3;
PeerCache cache = new PeerCache(Capacity, 100000);
DatanodeID dnId = new DatanodeID("192.168.0.1", "fakehostname", "fake_datanode_id"
, 100, 101, 102, 103);
HashMultiset <TestPeerCache.FakePeer> peers = HashMultiset.Create(Capacity);
for (int i = 0; i < Capacity; ++i)
{
TestPeerCache.FakePeer peer = new TestPeerCache.FakePeer(dnId, false);
peers.AddItem(peer);
cache.Put(dnId, peer);
}
// Check that all of the peers ended up in the cache
NUnit.Framework.Assert.AreEqual(Capacity, cache.Size());
while (!peers.IsEmpty())
{
Peer peer = cache.Get(dnId, false);
NUnit.Framework.Assert.IsTrue(peer != null);
NUnit.Framework.Assert.IsTrue(!peer.IsClosed());
peers.Remove(peer);
}
NUnit.Framework.Assert.AreEqual(0, cache.Size());
cache.Close();
}
public virtual void TestEviction()
{
int Capacity = 3;
PeerCache cache = new PeerCache(Capacity, 100000);
DatanodeID[] dnIds = new DatanodeID[Capacity + 1];
TestPeerCache.FakePeer[] peers = new TestPeerCache.FakePeer[Capacity + 1];
for (int i = 0; i < dnIds.Length; ++i)
{
dnIds[i] = new DatanodeID("192.168.0.1", "fakehostname_" + i, "fake_datanode_id_"
+ i, 100, 101, 102, 103);
peers[i] = new TestPeerCache.FakePeer(dnIds[i], false);
}
for (int i_1 = 0; i_1 < Capacity; ++i_1)
{
cache.Put(dnIds[i_1], peers[i_1]);
}
// Check that the peers are cached
NUnit.Framework.Assert.AreEqual(Capacity, cache.Size());
// Add another entry and check that the first entry was evicted
cache.Put(dnIds[Capacity], peers[Capacity]);
NUnit.Framework.Assert.AreEqual(Capacity, cache.Size());
NUnit.Framework.Assert.AreSame(null, cache.Get(dnIds[0], false));
// Make sure that the other entries are still there
for (int i_2 = 1; i_2 < Capacity; ++i_2)
{
Peer peer = cache.Get(dnIds[i_2], false);
NUnit.Framework.Assert.AreSame(peers[i_2], peer);
NUnit.Framework.Assert.IsTrue(!peer.IsClosed());
peer.Close();
}
NUnit.Framework.Assert.AreEqual(1, cache.Size());
cache.Close();
}
public virtual void TestAddAndRetrieve()
{
PeerCache cache = new PeerCache(3, 100000);
DatanodeID dnId = new DatanodeID("192.168.0.1", "fakehostname", "fake_datanode_id"
, 100, 101, 102, 103);
TestPeerCache.FakePeer peer = new TestPeerCache.FakePeer(dnId, false);
cache.Put(dnId, peer);
NUnit.Framework.Assert.IsTrue(!peer.IsClosed());
NUnit.Framework.Assert.AreEqual(1, cache.Size());
NUnit.Framework.Assert.AreEqual(peer, cache.Get(dnId, false));
NUnit.Framework.Assert.AreEqual(0, cache.Size());
cache.Close();
}
public virtual void TestExpiry()
{
int Capacity = 3;
int ExpiryPeriod = 10;
PeerCache cache = new PeerCache(Capacity, ExpiryPeriod);
DatanodeID[] dnIds = new DatanodeID[Capacity];
TestPeerCache.FakePeer[] peers = new TestPeerCache.FakePeer[Capacity];
for (int i = 0; i < Capacity; ++i)
{
dnIds[i] = new DatanodeID("192.168.0.1", "fakehostname_" + i, "fake_datanode_id",
100, 101, 102, 103);
peers[i] = new TestPeerCache.FakePeer(dnIds[i], false);
}
for (int i_1 = 0; i_1 < Capacity; ++i_1)
{
cache.Put(dnIds[i_1], peers[i_1]);
}
// Wait for the peers to expire
Sharpen.Thread.Sleep(ExpiryPeriod * 50);
NUnit.Framework.Assert.AreEqual(0, cache.Size());
// make sure that the peers were closed when they were expired
for (int i_2 = 0; i_2 < Capacity; ++i_2)
{
NUnit.Framework.Assert.IsTrue(peers[i_2].IsClosed());
}
// sleep for another second and see if
// the daemon thread runs fine on empty cache
Sharpen.Thread.Sleep(ExpiryPeriod * 50);
cache.Close();
}
/// <summary>Regression test for HDFS-3357.</summary>
/// <remarks>
/// Regression test for HDFS-3357. Check that the datanode is respecting
/// its configured keepalive timeout.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDatanodeRespectsKeepAliveTimeout()
{
Configuration clientConf = new Configuration(conf);
// Set a client socket cache expiry time much longer than
// the datanode-side expiration time.
long ClientExpiryMs = 60000L;
clientConf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, ClientExpiryMs
);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testDatanodeRespectsKeepAliveTimeout"
);
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI()
, clientConf);
PeerCache peerCache = ClientContext.GetFromConf(clientConf).GetPeerCache();
DFSTestUtil.CreateFile(fs, TestFile, 1L, (short)1, 0L);
// Clients that write aren't currently re-used.
NUnit.Framework.Assert.AreEqual(0, peerCache.Size());
AssertXceiverCount(0);
// Reads the file, so we should get a
// cached socket, and should have an xceiver on the other side.
DFSTestUtil.ReadFile(fs, TestFile);
NUnit.Framework.Assert.AreEqual(1, peerCache.Size());
AssertXceiverCount(1);
// Sleep for a bit longer than the keepalive timeout
// and make sure the xceiver died.
Sharpen.Thread.Sleep(DFSConfigKeys.DfsDatanodeSocketReuseKeepaliveDefault + 50);
AssertXceiverCount(0);
// The socket is still in the cache, because we don't
// notice that it's closed until we try to read
// from it again.
NUnit.Framework.Assert.AreEqual(1, peerCache.Size());
// Take it out of the cache - reading should
// give an EOF.
Peer peer = peerCache.Get(dn.GetDatanodeId(), false);
NUnit.Framework.Assert.IsNotNull(peer);
NUnit.Framework.Assert.AreEqual(-1, peer.GetInputStream().Read());
}
/// <exception cref="System.Exception"/>
public virtual void TestManyClosedSocketsInCache()
{
// Make a small file
Configuration clientConf = new Configuration(conf);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testManyClosedSocketsInCache");
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI()
, clientConf);
PeerCache peerCache = ClientContext.GetFromConf(clientConf).GetPeerCache();
DFSTestUtil.CreateFile(fs, TestFile, 1L, (short)1, 0L);
// Insert a bunch of dead sockets in the cache, by opening
// many streams concurrently, reading all of the data,
// and then closing them.
InputStream[] stms = new InputStream[5];
try
{
for (int i = 0; i < stms.Length; i++)
{
stms[i] = fs.Open(TestFile);
}
foreach (InputStream stm in stms)
{
IOUtils.CopyBytes(stm, new IOUtils.NullOutputStream(), 1024);
}
}
finally
{
IOUtils.Cleanup(null, stms);
}
NUnit.Framework.Assert.AreEqual(5, peerCache.Size());
// Let all the xceivers timeout
Sharpen.Thread.Sleep(1500);
AssertXceiverCount(0);
// Client side still has the sockets cached
NUnit.Framework.Assert.AreEqual(5, peerCache.Size());
// Reading should not throw an exception.
DFSTestUtil.ReadFile(fs, TestFile);
}
public virtual void TestDomainSocketPeers()
{
int Capacity = 3;
PeerCache cache = new PeerCache(Capacity, 100000);
DatanodeID dnId = new DatanodeID("192.168.0.1", "fakehostname", "fake_datanode_id"
, 100, 101, 102, 103);
HashMultiset <TestPeerCache.FakePeer> peers = HashMultiset.Create(Capacity);
for (int i = 0; i < Capacity; ++i)
{
TestPeerCache.FakePeer peer = new TestPeerCache.FakePeer(dnId, i == Capacity - 1);
peers.AddItem(peer);
cache.Put(dnId, peer);
}
// Check that all of the peers ended up in the cache
NUnit.Framework.Assert.AreEqual(Capacity, cache.Size());
// Test that get(requireDomainPeer=true) finds the peer with the
// domain socket.
Peer peer_1 = cache.Get(dnId, true);
NUnit.Framework.Assert.IsTrue(peer_1.GetDomainSocket() != null);
peers.Remove(peer_1);
// Test that get(requireDomainPeer=true) returns null when there are
// no more peers with domain sockets.
peer_1 = cache.Get(dnId, true);
NUnit.Framework.Assert.IsTrue(peer_1 == null);
// Check that all of the other peers ended up in the cache.
while (!peers.IsEmpty())
{
peer_1 = cache.Get(dnId, false);
NUnit.Framework.Assert.IsTrue(peer_1 != null);
NUnit.Framework.Assert.IsTrue(!peer_1.IsClosed());
peers.Remove(peer_1);
}
NUnit.Framework.Assert.AreEqual(0, cache.Size());
cache.Close();
}