/// <summary>
/// Test to verify that InterDatanode RPC timesout as expected when
/// the server DN does not respond.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestInterDNProtocolTimeout()
{
Org.Apache.Hadoop.Ipc.Server server = new TestInterDatanodeProtocol.TestServer(1,
true);
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port);
DatanodeInfo dInfo = new DatanodeInfo(fakeDnId);
InterDatanodeProtocol proxy = null;
try
{
proxy = DataNode.CreateInterDataNodeProtocolProxy(dInfo, conf, 500, false);
proxy.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock(new ExtendedBlock
("bpid", 1), null, 100));
NUnit.Framework.Assert.Fail("Expected SocketTimeoutException exception, but did not get."
);
}
finally
{
if (proxy != null)
{
RPC.StopProxy(proxy);
}
server.Stop();
}
}
public virtual void TestFinalizedRwrReplicas()
{
if (Log.IsDebugEnabled())
{
Log.Debug("Running " + GenericTestUtils.GetMethodName());
}
// rbw and finalized replicas have the same length
ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BlockId, ReplicaLen1, GenStamp
- 1, HdfsServerConstants.ReplicaState.Finalized);
ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BlockId, ReplicaLen1, GenStamp
- 2, HdfsServerConstants.ReplicaState.Rwr);
InterDatanodeProtocol dn1 = Org.Mockito.Mockito.Mock <InterDatanodeProtocol>();
InterDatanodeProtocol dn2 = Org.Mockito.Mockito.Mock <InterDatanodeProtocol>();
TestSyncReplicas(replica1, replica2, dn1, dn2, ReplicaLen1);
Org.Mockito.Mockito.Verify(dn1).UpdateReplicaUnderRecovery(block, RecoveryId, BlockId
, ReplicaLen1);
Org.Mockito.Mockito.Verify(dn2, Org.Mockito.Mockito.Never()).UpdateReplicaUnderRecovery
(block, RecoveryId, BlockId, ReplicaLen1);
// rbw replica has a different length from the finalized one
replica1 = new ReplicaRecoveryInfo(BlockId, ReplicaLen1, GenStamp - 1, HdfsServerConstants.ReplicaState
.Finalized);
replica2 = new ReplicaRecoveryInfo(BlockId, ReplicaLen2, GenStamp - 2, HdfsServerConstants.ReplicaState
.Rbw);
dn1 = Org.Mockito.Mockito.Mock <InterDatanodeProtocol>();
dn2 = Org.Mockito.Mockito.Mock <InterDatanodeProtocol>();
TestSyncReplicas(replica1, replica2, dn1, dn2, ReplicaLen1);
Org.Mockito.Mockito.Verify(dn1).UpdateReplicaUnderRecovery(block, RecoveryId, BlockId
, ReplicaLen1);
Org.Mockito.Mockito.Verify(dn2, Org.Mockito.Mockito.Never()).UpdateReplicaUnderRecovery
(block, RecoveryId, BlockId, ReplicaLen1);
}
public virtual void TestFinalizedReplicas()
{
if (Log.IsDebugEnabled())
{
Log.Debug("Running " + GenericTestUtils.GetMethodName());
}
ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BlockId, ReplicaLen1, GenStamp
- 1, HdfsServerConstants.ReplicaState.Finalized);
ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BlockId, ReplicaLen1, GenStamp
- 2, HdfsServerConstants.ReplicaState.Finalized);
InterDatanodeProtocol dn1 = Org.Mockito.Mockito.Mock <InterDatanodeProtocol>();
InterDatanodeProtocol dn2 = Org.Mockito.Mockito.Mock <InterDatanodeProtocol>();
TestSyncReplicas(replica1, replica2, dn1, dn2, ReplicaLen1);
Org.Mockito.Mockito.Verify(dn1).UpdateReplicaUnderRecovery(block, RecoveryId, BlockId
, ReplicaLen1);
Org.Mockito.Mockito.Verify(dn2).UpdateReplicaUnderRecovery(block, RecoveryId, BlockId
, ReplicaLen1);
// two finalized replicas have different length
replica1 = new ReplicaRecoveryInfo(BlockId, ReplicaLen1, GenStamp - 1, HdfsServerConstants.ReplicaState
.Finalized);
replica2 = new ReplicaRecoveryInfo(BlockId, ReplicaLen2, GenStamp - 2, HdfsServerConstants.ReplicaState
.Finalized);
try
{
TestSyncReplicas(replica1, replica2, dn1, dn2, ReplicaLen1);
NUnit.Framework.Assert.Fail("Two finalized replicas should not have different lengthes!"
);
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.StartsWith("Inconsistent size of finalized replicas. "
));
}
}
public virtual void TestRBW_RWRReplicas()
{
if (Log.IsDebugEnabled())
{
Log.Debug("Running " + GenericTestUtils.GetMethodName());
}
ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BlockId, ReplicaLen1, GenStamp
- 1, HdfsServerConstants.ReplicaState.Rbw);
ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BlockId, ReplicaLen1, GenStamp
- 2, HdfsServerConstants.ReplicaState.Rwr);
InterDatanodeProtocol dn1 = Org.Mockito.Mockito.Mock <InterDatanodeProtocol>();
InterDatanodeProtocol dn2 = Org.Mockito.Mockito.Mock <InterDatanodeProtocol>();
TestSyncReplicas(replica1, replica2, dn1, dn2, ReplicaLen1);
Org.Mockito.Mockito.Verify(dn1).UpdateReplicaUnderRecovery(block, RecoveryId, BlockId
, ReplicaLen1);
Org.Mockito.Mockito.Verify(dn2, Org.Mockito.Mockito.Never()).UpdateReplicaUnderRecovery
(block, RecoveryId, BlockId, ReplicaLen1);
}
/// <summary>Sync two replicas</summary>
/// <exception cref="System.IO.IOException"/>
private void TestSyncReplicas(ReplicaRecoveryInfo replica1, ReplicaRecoveryInfo replica2
, InterDatanodeProtocol dn1, InterDatanodeProtocol dn2, long expectLen)
{
DatanodeInfo[] locs = new DatanodeInfo[] { Org.Mockito.Mockito.Mock <DatanodeInfo>
(), Org.Mockito.Mockito.Mock <DatanodeInfo>() };
BlockRecoveryCommand.RecoveringBlock rBlock = new BlockRecoveryCommand.RecoveringBlock
(block, locs, RecoveryId);
AList <DataNode.BlockRecord> syncList = new AList <DataNode.BlockRecord>(2);
DataNode.BlockRecord record1 = new DataNode.BlockRecord(DFSTestUtil.GetDatanodeInfo
("1.2.3.4", "bogus", 1234), dn1, replica1);
DataNode.BlockRecord record2 = new DataNode.BlockRecord(DFSTestUtil.GetDatanodeInfo
("1.2.3.4", "bogus", 1234), dn2, replica2);
syncList.AddItem(record1);
syncList.AddItem(record2);
Org.Mockito.Mockito.When(dn1.UpdateReplicaUnderRecovery((ExtendedBlock)Matchers.AnyObject
(), Matchers.AnyLong(), Matchers.AnyLong(), Matchers.AnyLong())).ThenReturn("storage1"
);
Org.Mockito.Mockito.When(dn2.UpdateReplicaUnderRecovery((ExtendedBlock)Matchers.AnyObject
(), Matchers.AnyLong(), Matchers.AnyLong(), Matchers.AnyLong())).ThenReturn("storage2"
);
dn.SyncBlock(rBlock, syncList);
}
public InterDatanodeProtocolServerSideTranslatorPB(InterDatanodeProtocol impl)
{
this.impl = impl;
}
/// <summary>The following test first creates a file.</summary>
/// <remarks>
/// The following test first creates a file.
/// It verifies the block information from a datanode.
/// Then, it updates the block with new information and verifies again.
/// </remarks>
/// <param name="useDnHostname">whether DNs should connect to other DNs by hostname</param>
/// <exception cref="System.Exception"/>
private void CheckBlockMetaDataInfo(bool useDnHostname)
{
MiniDFSCluster cluster = null;
conf.SetBoolean(DFSConfigKeys.DfsDatanodeUseDnHostname, useDnHostname);
if (useDnHostname)
{
// Since the mini cluster only listens on the loopback we have to
// ensure the hostname used to access DNs maps to the loopback. We
// do this by telling the DN to advertise localhost as its hostname
// instead of the default hostname.
conf.Set(DFSConfigKeys.DfsDatanodeHostNameKey, "localhost");
}
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).CheckDataNodeHostConfig
(true).Build();
cluster.WaitActive();
//create a file
DistributedFileSystem dfs = cluster.GetFileSystem();
string filestr = "/foo";
Path filepath = new Path(filestr);
DFSTestUtil.CreateFile(dfs, filepath, 1024L, (short)3, 0L);
NUnit.Framework.Assert.IsTrue(dfs.Exists(filepath));
//get block info
LocatedBlock locatedblock = GetLastLocatedBlock(DFSClientAdapter.GetDFSClient(dfs
).GetNamenode(), filestr);
DatanodeInfo[] datanodeinfo = locatedblock.GetLocations();
NUnit.Framework.Assert.IsTrue(datanodeinfo.Length > 0);
//connect to a data node
DataNode datanode = cluster.GetDataNode(datanodeinfo[0].GetIpcPort());
InterDatanodeProtocol idp = DataNodeTestUtils.CreateInterDatanodeProtocolProxy(datanode
, datanodeinfo[0], conf, useDnHostname);
// Stop the block scanners.
datanode.GetBlockScanner().RemoveAllVolumeScanners();
//verify BlockMetaDataInfo
ExtendedBlock b = locatedblock.GetBlock();
InterDatanodeProtocol.Log.Info("b=" + b + ", " + b.GetType());
CheckMetaInfo(b, datanode);
long recoveryId = b.GetGenerationStamp() + 1;
idp.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock(b, locatedblock.
GetLocations(), recoveryId));
//verify updateBlock
ExtendedBlock newblock = new ExtendedBlock(b.GetBlockPoolId(), b.GetBlockId(), b.
GetNumBytes() / 2, b.GetGenerationStamp() + 1);
idp.UpdateReplicaUnderRecovery(b, recoveryId, b.GetBlockId(), newblock.GetNumBytes
());
CheckMetaInfo(newblock, datanode);
// Verify correct null response trying to init recovery for a missing block
ExtendedBlock badBlock = new ExtendedBlock("fake-pool", b.GetBlockId(), 0, 0);
NUnit.Framework.Assert.IsNull(idp.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock
(badBlock, locatedblock.GetLocations(), recoveryId)));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}