public virtual void TestChooseReplicaToDelete()
{
MiniDFSCluster cluster = null;
FileSystem fs = null;
try
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, SmallBlockSize);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
fs = cluster.GetFileSystem();
FSNamesystem namesystem = cluster.GetNamesystem();
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 300);
cluster.StartDataNodes(conf, 1, true, null, null, null);
DataNode lastDN = cluster.GetDataNodes()[3];
DatanodeRegistration dnReg = DataNodeTestUtils.GetDNRegistrationForBP(lastDN, namesystem
.GetBlockPoolId());
string lastDNid = dnReg.GetDatanodeUuid();
Path fileName = new Path("/foo2");
DFSTestUtil.CreateFile(fs, fileName, SmallFileLength, (short)4, 0L);
DFSTestUtil.WaitReplication(fs, fileName, (short)4);
// Wait for tolerable number of heartbeats plus one
DatanodeDescriptor nodeInfo = null;
long lastHeartbeat = 0;
long waitTime = DFSConfigKeys.DfsHeartbeatIntervalDefault * 1000 * (DFSConfigKeys
.DfsNamenodeTolerateHeartbeatMultiplierDefault + 1);
do
{
nodeInfo = namesystem.GetBlockManager().GetDatanodeManager().GetDatanode(dnReg);
lastHeartbeat = nodeInfo.GetLastUpdateMonotonic();
}while (Time.MonotonicNow() - lastHeartbeat < waitTime);
fs.SetReplication(fileName, (short)3);
BlockLocation[] locs = fs.GetFileBlockLocations(fs.GetFileStatus(fileName), 0, long.MaxValue
);
// All replicas for deletion should be scheduled on lastDN.
// And should not actually be deleted, because lastDN does not heartbeat.
namesystem.ReadLock();
ICollection <Block> dnBlocks = namesystem.GetBlockManager().excessReplicateMap[lastDNid
];
NUnit.Framework.Assert.AreEqual("Replicas on node " + lastDNid + " should have been deleted"
, SmallFileLength / SmallBlockSize, dnBlocks.Count);
namesystem.ReadUnlock();
foreach (BlockLocation location in locs)
{
NUnit.Framework.Assert.AreEqual("Block should still have 4 replicas", 4, location
.GetNames().Length);
}
}
finally
{
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
private static Configuration CreateConf()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsNamenodeAccesstimePrecisionKey, 1L);
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 10000L);
return(conf);
}
public static void SetUp()
{
// start a cluster
Configuration conf = new HdfsConfiguration();
// High value of replication interval
// so that blocks remain under-replicated
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1000);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L);
conf.SetLong(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1L);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Build();
cluster.WaitActive();
fileSys = cluster.GetFileSystem();
nnRpc = cluster.GetNameNodeRpc();
}
private static Configuration GetDefaultConf()
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsNamenodePathBasedCacheRefreshIntervalMs, 50);
conf.SetLong(DFSConfigKeys.DfsCachereportIntervalMsecKey, 250);
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
conf.SetLong(DFSConfigKeys.DfsDatanodeMaxLockedMemoryKey, TestFsDatasetCache.CacheCapacity
);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), "sock"
).GetAbsolutePath());
return(conf);
}
/// <exception cref="System.Exception"/>
public virtual void TestMinBlockSizeLimit()
{
long blockSize = 4096;
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsNamenodeMinBlockSizeKey, blockSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
// Try with min block size
fs.Create(new Path("/testmblock1"), true, 4096, (short)3, blockSize);
try
{
// Try with min block size - 1
fs.Create(new Path("/testmblock2"), true, 4096, (short)3, blockSize - 1);
System.Diagnostics.Debug.Assert(false, "Expected IOException after creating a file with small"
+ " blocks ");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("Specified block size is less", e);
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestFinalizedReplicas()
{
// bring up a cluster of 3
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024L);
conf.SetInt(DFSConfigKeys.DfsClientWritePacketSizeKey, 512);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
try
{
// test finalized replicas
string TopDir = "/test";
DFSTestUtil util = new DFSTestUtil.Builder().SetName("TestDatanodeRestart").SetNumFiles
(2).Build();
util.CreateFiles(fs, TopDir, (short)3);
util.WaitReplication(fs, TopDir, (short)3);
util.CheckFiles(fs, TopDir);
cluster.RestartDataNodes();
cluster.WaitActive();
util.CheckFiles(fs, TopDir);
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// The test verifies the number of outstanding replication requests for a
/// given DN shouldn't exceed the limit set by configuration property
/// dfs.namenode.replication.max-streams-hard-limit.
/// </summary>
/// <remarks>
/// The test verifies the number of outstanding replication requests for a
/// given DN shouldn't exceed the limit set by configuration property
/// dfs.namenode.replication.max-streams-hard-limit.
/// The test does the followings:
/// 1. Create a mini cluster with 2 DNs. Set large heartbeat interval so that
/// replication requests won't be picked by any DN right away.
/// 2. Create a file with 10 blocks and replication factor 2. Thus each
/// of the 2 DNs have one replica of each block.
/// 3. Add a DN to the cluster for later replication.
/// 4. Remove a DN that has data.
/// 5. Ask BlockManager to compute the replication work. This will assign
/// replication requests to the only DN that has data.
/// 6. Make sure the number of pending replication requests of that DN don't
/// exceed the limit.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestNumberOfBlocksToBeReplicated()
{
// 1 min timeout
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsNamenodeMinBlockSizeKey, 0);
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1);
conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, 1);
// Large value to make sure the pending replication request can stay in
// DatanodeDescriptor.replicateBlocks before test timeout.
conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 100);
// Make sure BlockManager can pull all blocks from UnderReplicatedBlocks via
// chooseUnderReplicatedBlocks at once.
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationWorkMultiplierPerIteration, 5);
int NumOfBlocks = 10;
short RepFactor = 2;
string FileName = "/testFile";
Path FilePath = new Path(FileName);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(RepFactor)
.Build();
try
{
// create a file with 10 blocks with a replication factor of 2
FileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, FilePath, NumOfBlocks, RepFactor, 1L);
DFSTestUtil.WaitReplication(fs, FilePath, RepFactor);
cluster.StartDataNodes(conf, 1, true, null, null, null, null);
BlockManager bm = cluster.GetNamesystem().GetBlockManager();
ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, FilePath);
IEnumerator <DatanodeStorageInfo> storageInfos = bm.blocksMap.GetStorages(b.GetLocalBlock
()).GetEnumerator();
DatanodeDescriptor firstDn = storageInfos.Next().GetDatanodeDescriptor();
DatanodeDescriptor secondDn = storageInfos.Next().GetDatanodeDescriptor();
bm.GetDatanodeManager().RemoveDatanode(firstDn);
NUnit.Framework.Assert.AreEqual(NumOfBlocks, bm.GetUnderReplicatedNotMissingBlocks
());
bm.ComputeDatanodeWork();
NUnit.Framework.Assert.IsTrue("The number of blocks to be replicated should be less than "
+ "or equal to " + bm.replicationStreamsHardLimit, secondDn.GetNumberOfBlocksToBeReplicated
() <= bm.replicationStreamsHardLimit);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestDeadDatanode()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 500);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L);
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
// wait for datanode to be marked live
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration reg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[0], poolId);
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), true, 20000);
// Shutdown and wait for datanode to be marked dead
dn.Shutdown();
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), false, 20000);
DatanodeProtocol dnp = cluster.GetNameNodeRpc();
ReceivedDeletedBlockInfo[] blocks = new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo
(new Block(0), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null) };
StorageReceivedDeletedBlocks[] storageBlocks = new StorageReceivedDeletedBlocks[]
{ new StorageReceivedDeletedBlocks(reg.GetDatanodeUuid(), blocks) };
// Ensure blockReceived call from dead datanode is rejected with IOException
try
{
dnp.BlockReceivedAndDeleted(reg, poolId, storageBlocks);
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure blockReport from dead datanode is rejected with IOException
StorageBlockReport[] report = new StorageBlockReport[] { new StorageBlockReport(new
DatanodeStorage(reg.GetDatanodeUuid()), BlockListAsLongs.Empty) };
try
{
dnp.BlockReport(reg, poolId, report, new BlockReportContext(1, 0, Runtime.NanoTime
()));
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure heartbeat from dead datanode is rejected with a command
// that asks datanode to register again
StorageReport[] rep = new StorageReport[] { new StorageReport(new DatanodeStorage
(reg.GetDatanodeUuid()), false, 0, 0, 0, 0) };
DatanodeCommand[] cmd = dnp.SendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null).GetCommands
();
NUnit.Framework.Assert.AreEqual(1, cmd.Length);
NUnit.Framework.Assert.AreEqual(cmd[0].GetAction(), RegisterCommand.Register.GetAction
());
}
public static HdfsConfiguration InitZeroCopyTest()
{
Assume.AssumeTrue(NativeIO.IsAvailable());
Assume.AssumeTrue(SystemUtils.IsOsUnix);
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
conf.SetInt(DFSConfigKeys.DfsClientMmapCacheSize, 3);
conf.SetLong(DFSConfigKeys.DfsClientMmapCacheTimeoutMs, 100);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), "TestRequestMmapAccess._PORT.sock"
).GetAbsolutePath());
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, true);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
conf.SetLong(DFSConfigKeys.DfsCachereportIntervalMsecKey, 1000);
conf.SetLong(DFSConfigKeys.DfsNamenodePathBasedCacheRefreshIntervalMs, 1000);
return(conf);
}
public virtual void TestRecoverReplicas()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024L);
conf.SetInt(DFSConfigKeys.DfsClientWritePacketSizeKey, 512);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
try
{
FileSystem fs = cluster.GetFileSystem();
for (int i = 0; i < 4; i++)
{
Path fileName = new Path("/test" + i);
DFSTestUtil.CreateFile(fs, fileName, 1, (short)1, 0L);
DFSTestUtil.WaitReplication(fs, fileName, (short)1);
}
string bpid = cluster.GetNamesystem().GetBlockPoolId();
DataNode dn = cluster.GetDataNodes()[0];
IEnumerator <ReplicaInfo> replicasItor = Dataset(dn).volumeMap.Replicas(bpid).GetEnumerator
();
ReplicaInfo replica = replicasItor.Next();
CreateUnlinkTmpFile(replica, true, true);
// rename block file
CreateUnlinkTmpFile(replica, false, true);
// rename meta file
replica = replicasItor.Next();
CreateUnlinkTmpFile(replica, true, false);
// copy block file
CreateUnlinkTmpFile(replica, false, false);
// copy meta file
replica = replicasItor.Next();
CreateUnlinkTmpFile(replica, true, true);
// rename block file
CreateUnlinkTmpFile(replica, false, false);
// copy meta file
cluster.RestartDataNodes();
cluster.WaitActive();
dn = cluster.GetDataNodes()[0];
// check volumeMap: 4 finalized replica
ICollection <ReplicaInfo> replicas = Dataset(dn).volumeMap.Replicas(bpid);
NUnit.Framework.Assert.AreEqual(4, replicas.Count);
replicasItor = replicas.GetEnumerator();
while (replicasItor.HasNext())
{
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.ReplicaState.Finalized, replicasItor
.Next().GetState());
}
}
finally
{
cluster.Shutdown();
}
}
/*
* Return a configuration object with low timeouts for testing and
* a topology script set (which enables rack awareness).
*/
private Configuration GetConf()
{
Configuration conf = new HdfsConfiguration();
// Lower the heart beat interval so the NN quickly learns of dead
// or decommissioned DNs and the NN issues replication and invalidation
// commands quickly (as replies to heartbeats)
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L);
// Have the NN ReplicationMonitor compute the replication and
// invalidation commands to send DNs every second.
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1);
// Have the NN check for pending replications every second so it
// quickly schedules additional replicas as they are identified.
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, 1);
// The DNs report blocks every second.
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L);
// Indicates we have multiple racks
conf.Set(DFSConfigKeys.NetTopologyScriptFileNameKey, "xyz");
return(conf);
}
internal static Configuration CreateConf()
{
Configuration conf = new HdfsConfiguration();
TestBalancer.InitConf(conf);
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
conf.Set(CommonConfigurationKeysPublic.NetTopologyImplKey, typeof(NetworkTopologyWithNodeGroup
).FullName);
conf.Set(DFSConfigKeys.DfsBlockReplicatorClassnameKey, typeof(BlockPlacementPolicyWithNodeGroup
).FullName);
return(conf);
}
public virtual void TestWithAllCorruptReplicas()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L);
conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString
(2));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
FileSystem fs = cluster.GetFileSystem();
FSNamesystem namesystem = cluster.GetNamesystem();
try
{
Path fileName = new Path("/foo1");
DFSTestUtil.CreateFile(fs, fileName, 2, (short)3, 0L);
DFSTestUtil.WaitReplication(fs, fileName, (short)3);
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName);
CorruptBlock(cluster, fs, fileName, 0, block);
CorruptBlock(cluster, fs, fileName, 1, block);
CorruptBlock(cluster, fs, fileName, 2, block);
// wait for 3 seconds so that all block reports are processed.
try
{
Sharpen.Thread.Sleep(3000);
}
catch (Exception)
{
}
NUnit.Framework.Assert.AreEqual(0, CountReplicas(namesystem, block).LiveReplicas(
));
NUnit.Framework.Assert.AreEqual(3, CountReplicas(namesystem, block).CorruptReplicas
());
namesystem.SetReplication(fileName.ToString(), (short)1);
// wait for 3 seconds so that all block reports are processed.
try
{
Sharpen.Thread.Sleep(3000);
}
catch (Exception)
{
}
NUnit.Framework.Assert.AreEqual(0, CountReplicas(namesystem, block).LiveReplicas(
));
NUnit.Framework.Assert.AreEqual(3, CountReplicas(namesystem, block).CorruptReplicas
());
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestMaxBlocksPerFileLimit()
{
Configuration conf = new HdfsConfiguration();
// Make a small block size and a low limit
long blockSize = 4096;
long numBlocks = 2;
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, blockSize);
conf.SetLong(DFSConfigKeys.DfsNamenodeMaxBlocksPerFileKey, numBlocks);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
HdfsDataOutputStream fout = (HdfsDataOutputStream)fs.Create(new Path("/testmaxfilelimit"
));
try
{
// Write maximum number of blocks
fout.Write(new byte[(int)blockSize * (int)numBlocks]);
fout.Hflush();
// Try to write one more block
try
{
fout.Write(new byte[1]);
fout.Hflush();
System.Diagnostics.Debug.Assert(false, "Expected IOException after writing too many blocks"
);
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("File has reached the limit" + " on maximum number of"
, e);
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void Setup()
{
StaticMapping.ResetMap();
Configuration conf = new HdfsConfiguration();
string[] racks = new string[] { "/RACK0", "/RACK0", "/RACK2", "/RACK3", "/RACK2" };
string[] hosts = new string[] { "/host0", "/host1", "/host2", "/host3", "/host4" };
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, DefaultBlockSize / 2);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(5).Racks(racks).Hosts(hosts
).Build();
cluster.WaitActive();
nameNodeRpc = cluster.GetNameNodeRpc();
namesystem = cluster.GetNamesystem();
perm = new PermissionStatus("TestDefaultBlockPlacementPolicy", null, FsPermission
.GetDefault());
}
// test rbw replicas persist across DataNode restarts
/// <exception cref="System.IO.IOException"/>
public virtual void TestRbwReplicas()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024L);
conf.SetInt(DFSConfigKeys.DfsClientWritePacketSizeKey, 512);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
cluster.WaitActive();
try
{
TestRbwReplicas(cluster, false);
TestRbwReplicas(cluster, true);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestStartNNWithTrashEmptier()
{
MiniDFSCluster cluster = null;
Configuration conf = new HdfsConfiguration();
// enable both trash emptier and dropping response
conf.SetLong("fs.trash.interval", 360);
conf.SetInt(DFSConfigKeys.DfsClientTestDropNamenodeResponseNumKey, 2);
try
{
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(0).Build();
cluster.WaitActive();
cluster.TransitionToActive(0);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestByAddingAnExtraDataNode()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L);
conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString
(2));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build();
FileSystem fs = cluster.GetFileSystem();
FSNamesystem namesystem = cluster.GetNamesystem();
MiniDFSCluster.DataNodeProperties dnPropsFourth = cluster.StopDataNode(3);
try
{
Path fileName = new Path("/foo1");
DFSTestUtil.CreateFile(fs, fileName, 2, (short)3, 0L);
DFSTestUtil.WaitReplication(fs, fileName, (short)3);
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName);
CorruptBlock(cluster, fs, fileName, 0, block);
DFSTestUtil.WaitReplication(fs, fileName, (short)2);
NUnit.Framework.Assert.AreEqual(2, CountReplicas(namesystem, block).LiveReplicas(
));
NUnit.Framework.Assert.AreEqual(1, CountReplicas(namesystem, block).CorruptReplicas
());
cluster.RestartDataNode(dnPropsFourth);
DFSTestUtil.WaitReplication(fs, fileName, (short)3);
NUnit.Framework.Assert.AreEqual(3, CountReplicas(namesystem, block).LiveReplicas(
));
NUnit.Framework.Assert.AreEqual(0, CountReplicas(namesystem, block).CorruptReplicas
());
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestPendingAndInvalidate()
{
Configuration Conf = new HdfsConfiguration();
Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
Conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, DfsReplicationInterval);
Conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, DfsReplicationInterval
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(DatanodeCount
).Build();
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
BlockManager bm = namesystem.GetBlockManager();
DistributedFileSystem fs = cluster.GetFileSystem();
try
{
// 1. create a file
Path filePath = new Path("/tmp.txt");
DFSTestUtil.CreateFile(fs, filePath, 1024, (short)3, 0L);
// 2. disable the heartbeats
foreach (DataNode dn in cluster.GetDataNodes())
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(dn, true);
}
// 3. mark a couple of blocks as corrupt
LocatedBlock block = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), filePath
.ToString(), 0, 1).Get(0);
cluster.GetNamesystem().WriteLock();
try
{
bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[0], "STORAGE_ID"
, "TEST");
bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[1], "STORAGE_ID"
, "TEST");
}
finally
{
cluster.GetNamesystem().WriteUnlock();
}
BlockManagerTestUtil.ComputeAllPendingWork(bm);
BlockManagerTestUtil.UpdateState(bm);
NUnit.Framework.Assert.AreEqual(bm.GetPendingReplicationBlocksCount(), 1L);
NUnit.Framework.Assert.AreEqual(bm.pendingReplications.GetNumReplicas(block.GetBlock
().GetLocalBlock()), 2);
// 4. delete the file
fs.Delete(filePath, true);
// retry at most 10 times, each time sleep for 1s. Note that 10s is much
// less than the default pending record timeout (5~10min)
int retries = 10;
long pendingNum = bm.GetPendingReplicationBlocksCount();
while (pendingNum != 0 && retries-- > 0)
{
Sharpen.Thread.Sleep(1000);
// let NN do the deletion
BlockManagerTestUtil.UpdateState(bm);
pendingNum = bm.GetPendingReplicationBlocksCount();
}
NUnit.Framework.Assert.AreEqual(pendingNum, 0L);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestBlockReceived()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build();
cluster.WaitActive();
DistributedFileSystem hdfs = cluster.GetFileSystem();
FSNamesystem fsn = cluster.GetNamesystem();
BlockManager blkManager = fsn.GetBlockManager();
string file = "/tmp.txt";
Path filePath = new Path(file);
short replFactor = 1;
DFSTestUtil.CreateFile(hdfs, filePath, 1024L, replFactor, 0);
// temporarily stop the heartbeat
AList <DataNode> datanodes = cluster.GetDataNodes();
for (int i = 0; i < DatanodeCount; i++)
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i], true);
}
hdfs.SetReplication(filePath, (short)DatanodeCount);
BlockManagerTestUtil.ComputeAllPendingWork(blkManager);
NUnit.Framework.Assert.AreEqual(1, blkManager.pendingReplications.Size());
INodeFile fileNode = fsn.GetFSDirectory().GetINode4Write(file).AsFile();
Block[] blocks = fileNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(DatanodeCount - 1, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
LocatedBlock locatedBlock = hdfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
DatanodeInfo existingDn = (locatedBlock.GetLocations())[0];
int reportDnNum = 0;
string poolId = cluster.GetNamesystem().GetBlockPoolId();
// let two datanodes (other than the one that already has the data) to
// report to NN
for (int i_1 = 0; i_1 < DatanodeCount && reportDnNum < 2; i_1++)
{
if (!datanodes[i_1].GetDatanodeId().Equals(existingDn))
{
DatanodeRegistration dnR = datanodes[i_1].GetDNRegistrationForBP(poolId);
StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new
StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo
[] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, string.Empty) }) };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report);
reportDnNum++;
}
}
NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
// let the same datanodes report again
for (int i_2 = 0; i_2 < DatanodeCount && reportDnNum < 2; i_2++)
{
if (!datanodes[i_2].GetDatanodeId().Equals(existingDn))
{
DatanodeRegistration dnR = datanodes[i_2].GetDNRegistrationForBP(poolId);
StorageReceivedDeletedBlocks[] report = new StorageReceivedDeletedBlocks[] { new
StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo
[] { new ReceivedDeletedBlockInfo(blocks[0], ReceivedDeletedBlockInfo.BlockStatus
.ReceivedBlock, string.Empty) }) };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dnR, poolId, report);
reportDnNum++;
}
}
NUnit.Framework.Assert.AreEqual(DatanodeCount - 3, blkManager.pendingReplications
.GetNumReplicas(blocks[0]));
// re-enable heartbeat for the datanode that has data
for (int i_3 = 0; i_3 < DatanodeCount; i_3++)
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(datanodes[i_3], false);
DataNodeTestUtils.TriggerHeartbeat(datanodes[i_3]);
}
Sharpen.Thread.Sleep(5000);
NUnit.Framework.Assert.AreEqual(0, blkManager.pendingReplications.Size());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestProcessPendingReplications()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Timeout);
MiniDFSCluster cluster = null;
Block block;
BlockInfoContiguous blockInfo;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build();
cluster.WaitActive();
FSNamesystem fsn = cluster.GetNamesystem();
BlockManager blkManager = fsn.GetBlockManager();
PendingReplicationBlocks pendingReplications = blkManager.pendingReplications;
UnderReplicatedBlocks neededReplications = blkManager.neededReplications;
BlocksMap blocksMap = blkManager.blocksMap;
//
// Add 1 block to pendingReplications with GenerationStamp = 0.
//
block = new Block(1, 1, 0);
blockInfo = new BlockInfoContiguous(block, (short)3);
pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil
.CreateDatanodeStorageInfos(1)));
BlockCollection bc = Org.Mockito.Mockito.Mock <BlockCollection>();
Org.Mockito.Mockito.DoReturn((short)3).When(bc).GetBlockReplication();
// Place into blocksmap with GenerationStamp = 1
blockInfo.SetGenerationStamp(1);
blocksMap.AddBlockCollection(blockInfo, bc);
NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 1, pendingReplications
.Size());
// Add a second block to pendingReplications that has no
// corresponding entry in blocksmap
block = new Block(2, 2, 0);
pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil
.CreateDatanodeStorageInfos(1)));
// verify 2 blocks in pendingReplications
NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 2, pendingReplications
.Size());
//
// Wait for everything to timeout.
//
while (pendingReplications.Size() > 0)
{
try
{
Sharpen.Thread.Sleep(100);
}
catch (Exception)
{
}
}
//
// Verify that block moves to neededReplications
//
while (neededReplications.Size() == 0)
{
try
{
Sharpen.Thread.Sleep(100);
}
catch (Exception)
{
}
}
// Verify that the generation stamp we will try to replicate
// is now 1
foreach (Block b in neededReplications)
{
NUnit.Framework.Assert.AreEqual("Generation stamp is 1 ", 1, b.GetGenerationStamp
());
}
// Verify size of neededReplications is exactly 1.
NUnit.Framework.Assert.AreEqual("size of neededReplications is 1 ", 1, neededReplications
.Size());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestFileLimit()
{
Configuration conf = new HdfsConfiguration();
int maxObjects = 5;
conf.SetLong(DFSConfigKeys.DfsNamenodeMaxObjectsKey, maxObjects);
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L);
conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
int currentNodes = 0;
if (simulatedStorage)
{
SimulatedFSDataset.SetFactory(conf);
}
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
FSNamesystem namesys = cluster.GetNamesystem();
try
{
//
// check that / exists
//
Path path = new Path("/");
NUnit.Framework.Assert.IsTrue("/ should be a directory", fs.GetFileStatus(path).IsDirectory
());
currentNodes = 1;
// root inode
// verify that we can create the specified number of files. We leave
// one for the "/". Each file takes an inode and a block.
//
for (int i = 0; i < maxObjects / 2; i++)
{
Path file = new Path("/filestatus" + i);
CreateFile(fs, file);
System.Console.Out.WriteLine("Created file " + file);
currentNodes += 2;
}
// two more objects for this creation.
// verify that creating another file fails
bool hitException = false;
try
{
Path file = new Path("/filestatus");
CreateFile(fs, file);
System.Console.Out.WriteLine("Created file " + file);
}
catch (IOException)
{
hitException = true;
}
NUnit.Framework.Assert.IsTrue("Was able to exceed file limit", hitException);
// delete one file
Path file0 = new Path("/filestatus0");
fs.Delete(file0, true);
System.Console.Out.WriteLine("Deleted file " + file0);
currentNodes -= 2;
// wait for number of blocks to decrease
WaitForLimit(namesys, currentNodes);
// now, we shud be able to create a new file
CreateFile(fs, file0);
System.Console.Out.WriteLine("Created file " + file0 + " again.");
currentNodes += 2;
// delete the file again
file0 = new Path("/filestatus0");
fs.Delete(file0, true);
System.Console.Out.WriteLine("Deleted file " + file0 + " again.");
currentNodes -= 2;
// wait for number of blocks to decrease
WaitForLimit(namesys, currentNodes);
// create two directories in place of the file that we deleted
Path dir = new Path("/dir0/dir1");
fs.Mkdirs(dir);
System.Console.Out.WriteLine("Created directories " + dir);
currentNodes += 2;
WaitForLimit(namesys, currentNodes);
// verify that creating another directory fails
hitException = false;
try
{
fs.Mkdirs(new Path("dir.fail"));
System.Console.Out.WriteLine("Created directory should not have succeeded.");
}
catch (IOException)
{
hitException = true;
}
NUnit.Framework.Assert.IsTrue("Was able to exceed dir limit", hitException);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
public virtual void TestWaitForRegistrationOnRestart()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsDatanodeBpReadyTimeoutKey, 5);
conf.SetInt(DFSConfigKeys.DfsClientSocketTimeoutKey, 5000);
// This makes the datanode appear registered to the NN, but it won't be
// able to get to the saved dn reg internally.
DataNodeFaultInjector dnFaultInjector = new _DataNodeFaultInjector_224();
DataNodeFaultInjector oldDnInjector = DataNodeFaultInjector.Get();
DataNodeFaultInjector.Set(dnFaultInjector);
MiniDFSCluster cluster = null;
long start = 0;
Path file = new Path("/reg");
try
{
int numDNs = 1;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDNs).Build();
cluster.WaitActive();
start = Runtime.CurrentTimeMillis();
FileSystem fileSys = cluster.GetFileSystem();
try
{
DFSTestUtil.CreateFile(fileSys, file, 10240L, (short)1, 0L);
// It is a bug if this does not fail.
throw new IOException("Did not fail!");
}
catch (RemoteException e)
{
long elapsed = Runtime.CurrentTimeMillis() - start;
// timers have at-least semantics, so it should be at least 5 seconds.
if (elapsed < 5000 || elapsed > 10000)
{
throw new IOException(elapsed + " seconds passed.", e);
}
}
DataNodeFaultInjector.Set(oldDnInjector);
// this should succeed now.
DFSTestUtil.CreateFile(fileSys, file, 10240L, (short)1, 0L);
// turn it back to under-construction, so that the client calls
// getReplicaVisibleLength() rpc method against the datanode.
fileSys.Append(file);
// back to simulating unregistered node.
DataNodeFaultInjector.Set(dnFaultInjector);
byte[] buffer = new byte[8];
start = Runtime.CurrentTimeMillis();
try
{
fileSys.Open(file).Read(0L, buffer, 0, 1);
throw new IOException("Did not fail!");
}
catch (IOException e)
{
long elapsed = Runtime.CurrentTimeMillis() - start;
if (e.Message.Contains("readBlockLength"))
{
throw new IOException("Failed, but with unexpected exception:", e);
}
// timers have at-least semantics, so it should be at least 5 seconds.
if (elapsed < 5000 || elapsed > 10000)
{
throw new IOException(elapsed + " seconds passed.", e);
}
}
DataNodeFaultInjector.Set(oldDnInjector);
fileSys.Open(file).Read(0L, buffer, 0, 1);
}
finally
{
DataNodeFaultInjector.Set(oldDnInjector);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void Test2GBMmapLimit()
{
Assume.AssumeTrue(BlockReaderTestUtil.ShouldTestLargeFiles());
HdfsConfiguration conf = InitZeroCopyTest();
long TestFileLength = 2469605888L;
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "NULL");
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, TestFileLength);
MiniDFSCluster cluster = null;
Path TestPath = new Path("/a");
string Context = "test2GBMmapLimit";
conf.Set(DFSConfigKeys.DfsClientContext, Context);
FSDataInputStream fsIn = null;
FSDataInputStream fsIn2 = null;
ByteBuffer buf1 = null;
ByteBuffer buf2 = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, unchecked ((int)(0xB
)));
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
fsIn = fs.Open(TestPath);
buf1 = fsIn.Read(null, 1, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(1, buf1.Remaining());
fsIn.ReleaseBuffer(buf1);
buf1 = null;
fsIn.Seek(2147483640L);
buf1 = fsIn.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(7, buf1.Remaining());
NUnit.Framework.Assert.AreEqual(int.MaxValue, buf1.Limit());
fsIn.ReleaseBuffer(buf1);
buf1 = null;
NUnit.Framework.Assert.AreEqual(2147483647L, fsIn.GetPos());
try
{
buf1 = fsIn.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.Fail("expected UnsupportedOperationException");
}
catch (NotSupportedException)
{
}
// expected; can't read past 2GB boundary.
fsIn.Close();
fsIn = null;
// Now create another file with normal-sized blocks, and verify we
// can read past 2GB
Path TestPath2 = new Path("/b");
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 268435456L);
DFSTestUtil.CreateFile(fs, TestPath2, 1024 * 1024, TestFileLength, 268435456L, (short
)1, unchecked ((int)(0xA)));
fsIn2 = fs.Open(TestPath2);
fsIn2.Seek(2147483640L);
buf2 = fsIn2.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(8, buf2.Remaining());
NUnit.Framework.Assert.AreEqual(2147483648L, fsIn2.GetPos());
fsIn2.ReleaseBuffer(buf2);
buf2 = null;
buf2 = fsIn2.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(1024, buf2.Remaining());
NUnit.Framework.Assert.AreEqual(2147484672L, fsIn2.GetPos());
fsIn2.ReleaseBuffer(buf2);
buf2 = null;
}
finally
{
if (buf1 != null)
{
fsIn.ReleaseBuffer(buf1);
}
if (buf2 != null)
{
fsIn2.ReleaseBuffer(buf2);
}
IOUtils.Cleanup(null, fsIn, fsIn2);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestVolumeSize()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
// Set aside fifth of the total capacity as reserved
long reserved = 10000;
conf.SetLong(DFSConfigKeys.DfsDatanodeDuReservedKey, reserved);
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
DatanodeManager dm = cluster.GetNamesystem().GetBlockManager().GetDatanodeManager
();
// Ensure the data reported for each data node is right
IList <DatanodeDescriptor> live = new AList <DatanodeDescriptor>();
IList <DatanodeDescriptor> dead = new AList <DatanodeDescriptor>();
dm.FetchDatanodes(live, dead, false);
NUnit.Framework.Assert.IsTrue(live.Count == 1);
long used;
long remaining;
long configCapacity;
long nonDFSUsed;
long bpUsed;
float percentUsed;
float percentRemaining;
float percentBpUsed;
foreach (DatanodeDescriptor datanode in live)
{
used = datanode.GetDfsUsed();
remaining = datanode.GetRemaining();
nonDFSUsed = datanode.GetNonDfsUsed();
configCapacity = datanode.GetCapacity();
percentUsed = datanode.GetDfsUsedPercent();
percentRemaining = datanode.GetRemainingPercent();
bpUsed = datanode.GetBlockPoolUsed();
percentBpUsed = datanode.GetBlockPoolUsedPercent();
Log.Info("Datanode configCapacity " + configCapacity + " used " + used + " non DFS used "
+ nonDFSUsed + " remaining " + remaining + " perentUsed " + percentUsed + " percentRemaining "
+ percentRemaining);
NUnit.Framework.Assert.IsTrue(configCapacity == (used + remaining + nonDFSUsed));
NUnit.Framework.Assert.IsTrue(percentUsed == DFSUtil.GetPercentUsed(used, configCapacity
));
NUnit.Framework.Assert.IsTrue(percentRemaining == DFSUtil.GetPercentRemaining(remaining
, configCapacity));
NUnit.Framework.Assert.IsTrue(percentBpUsed == DFSUtil.GetPercentUsed(bpUsed, configCapacity
));
}
DF df = new DF(new FilePath(cluster.GetDataDirectory()), conf);
//
// Currently two data directories are created by the data node
// in the MiniDFSCluster. This results in each data directory having
// capacity equals to the disk capacity of the data directory.
// Hence the capacity reported by the data node is twice the disk space
// the disk capacity
//
// So multiply the disk capacity and reserved space by two
// for accommodating it
//
int numOfDataDirs = 2;
long diskCapacity = numOfDataDirs * df.GetCapacity();
reserved *= numOfDataDirs;
configCapacity = namesystem.GetCapacityTotal();
used = namesystem.GetCapacityUsed();
nonDFSUsed = namesystem.GetNonDfsUsedSpace();
remaining = namesystem.GetCapacityRemaining();
percentUsed = namesystem.GetPercentUsed();
percentRemaining = namesystem.GetPercentRemaining();
bpUsed = namesystem.GetBlockPoolUsedSpace();
percentBpUsed = namesystem.GetPercentBlockPoolUsed();
Log.Info("Data node directory " + cluster.GetDataDirectory());
Log.Info("Name node diskCapacity " + diskCapacity + " configCapacity " + configCapacity
+ " reserved " + reserved + " used " + used + " remaining " + remaining + " nonDFSUsed "
+ nonDFSUsed + " remaining " + remaining + " percentUsed " + percentUsed + " percentRemaining "
+ percentRemaining + " bpUsed " + bpUsed + " percentBpUsed " + percentBpUsed);
// Ensure new total capacity reported excludes the reserved space
NUnit.Framework.Assert.IsTrue(configCapacity == diskCapacity - reserved);
// Ensure new total capacity reported excludes the reserved space
NUnit.Framework.Assert.IsTrue(configCapacity == (used + remaining + nonDFSUsed));
// Ensure percent used is calculated based on used and present capacity
NUnit.Framework.Assert.IsTrue(percentUsed == DFSUtil.GetPercentUsed(used, configCapacity
));
// Ensure percent used is calculated based on used and present capacity
NUnit.Framework.Assert.IsTrue(percentBpUsed == DFSUtil.GetPercentUsed(bpUsed, configCapacity
));
// Ensure percent used is calculated based on used and present capacity
NUnit.Framework.Assert.IsTrue(percentRemaining == ((float)remaining * 100.0f) / (
float)configCapacity);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
private void TestTriggerBlockReport(bool incremental)
{
Configuration conf = new HdfsConfiguration();
// Set a really long value for dfs.blockreport.intervalMsec and
// dfs.heartbeat.interval, so that incremental block reports and heartbeats
// won't be sent during this test unless they're triggered
// manually.
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 10800000L);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1080L);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
DatanodeProtocolClientSideTranslatorPB spy = DataNodeTestUtils.SpyOnBposToNN(cluster
.GetDataNodes()[0], cluster.GetNameNode());
DFSTestUtil.CreateFile(fs, new Path("/abc"), 16, (short)1, 1L);
// We should get 1 incremental block report.
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Timeout(60000).Times(1)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
// We should not receive any more incremental or incremental block reports,
// since the interval we configured is so long.
for (int i = 0; i < 3; i++)
{
Sharpen.Thread.Sleep(10);
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Times(0)).BlockReport(Matchers.Any
<DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageBlockReport[]
>(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Times(1)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
}
// Create a fake block deletion notification on the DataNode.
// This will be sent with the next incremental block report.
ReceivedDeletedBlockInfo rdbi = new ReceivedDeletedBlockInfo(new Block(5678, 512,
1000), ReceivedDeletedBlockInfo.BlockStatus.DeletedBlock, null);
DataNode datanode = cluster.GetDataNodes()[0];
BPServiceActor actor = datanode.GetAllBpOs()[0].GetBPServiceActors()[0];
string storageUuid = datanode.GetFSDataset().GetVolumes()[0].GetStorageID();
actor.NotifyNamenodeDeletedBlock(rdbi, storageUuid);
// Manually trigger a block report.
datanode.TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental(incremental
).Build());
// triggerBlockReport returns before the block report is
// actually sent. Wait for it to be sent here.
if (incremental)
{
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Timeout(60000).Times(2)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
}
else
{
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Timeout(60000)).BlockReport(Matchers.Any
<DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageBlockReport[]
>(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
}
cluster.Shutdown();
}
public virtual void TestPurgingWithNameEditsDirAfterFailure()
{
MiniDFSCluster cluster = null;
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsNamenodeNumExtraEditsRetainedKey, 0);
FilePath sd0 = new FilePath(TestRootDir, "nn0");
FilePath sd1 = new FilePath(TestRootDir, "nn1");
FilePath cd0 = new FilePath(sd0, "current");
FilePath cd1 = new FilePath(sd1, "current");
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, Joiner.On(",").Join(sd0, sd1));
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs(false
).Format(true).Build();
NameNode nn = cluster.GetNameNode();
DoSaveNamespace(nn);
Log.Info("After first save, images 0 and 2 should exist in both dirs");
GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName
(0), NNStorage.GetImageFileName(2));
GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName
(0), NNStorage.GetImageFileName(2));
GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetFinalizedEditsFileName
(1, 2), NNStorage.GetInProgressEditsFileName(3));
GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName
(1, 2), NNStorage.GetInProgressEditsFileName(3));
DoSaveNamespace(nn);
Log.Info("After second save, image 0 should be purged, " + "and image 4 should exist in both."
);
GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName
(2), NNStorage.GetImageFileName(4));
GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName
(2), NNStorage.GetImageFileName(4));
GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetFinalizedEditsFileName
(3, 4), NNStorage.GetInProgressEditsFileName(5));
GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName
(3, 4), NNStorage.GetInProgressEditsFileName(5));
Log.Info("Failing first storage dir by chmodding it");
NUnit.Framework.Assert.AreEqual(0, FileUtil.Chmod(cd0.GetAbsolutePath(), "000"));
DoSaveNamespace(nn);
Log.Info("Restoring accessibility of first storage dir");
NUnit.Framework.Assert.AreEqual(0, FileUtil.Chmod(cd0.GetAbsolutePath(), "755"));
Log.Info("nothing should have been purged in first storage dir");
GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName
(2), NNStorage.GetImageFileName(4));
GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetFinalizedEditsFileName
(3, 4), NNStorage.GetInProgressEditsFileName(5));
Log.Info("fsimage_2 should be purged in second storage dir");
GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName
(4), NNStorage.GetImageFileName(6));
GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName
(5, 6), NNStorage.GetInProgressEditsFileName(7));
Log.Info("On next save, we should purge logs from the failed dir," + " but not images, since the image directory is in failed state."
);
DoSaveNamespace(nn);
GenericTestUtils.AssertGlobEquals(cd1, "fsimage_\\d*", NNStorage.GetImageFileName
(6), NNStorage.GetImageFileName(8));
GenericTestUtils.AssertGlobEquals(cd1, "edits_.*", NNStorage.GetFinalizedEditsFileName
(7, 8), NNStorage.GetInProgressEditsFileName(9));
GenericTestUtils.AssertGlobEquals(cd0, "fsimage_\\d*", NNStorage.GetImageFileName
(2), NNStorage.GetImageFileName(4));
GenericTestUtils.AssertGlobEquals(cd0, "edits_.*", NNStorage.GetInProgressEditsFileName
(9));
}
finally
{
FileUtil.Chmod(cd0.GetAbsolutePath(), "755");
Log.Info("Shutting down...");
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestBlockReplacement()
{
Configuration Conf = new HdfsConfiguration();
string[] InitialRacks = new string[] { "/RACK0", "/RACK1", "/RACK2" };
string[] NewRacks = new string[] { "/RACK2" };
short ReplicationFactor = (short)3;
int DefaultBlockSize = 1024;
Random r = new Random();
Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
Conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, DefaultBlockSize / 2);
Conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 500);
cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(ReplicationFactor).Racks(
InitialRacks).Build();
try
{
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
Path fileName = new Path("/tmp.txt");
// create a file with one block
DFSTestUtil.CreateFile(fs, fileName, DefaultBlockSize, ReplicationFactor, r.NextLong
());
DFSTestUtil.WaitReplication(fs, fileName, ReplicationFactor);
// get all datanodes
IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort());
DFSClient client = new DFSClient(addr, Conf);
IList <LocatedBlock> locatedBlocks = client.GetNamenode().GetBlockLocations("/tmp.txt"
, 0, DefaultBlockSize).GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, locatedBlocks.Count);
LocatedBlock block = locatedBlocks[0];
DatanodeInfo[] oldNodes = block.GetLocations();
NUnit.Framework.Assert.AreEqual(oldNodes.Length, 3);
ExtendedBlock b = block.GetBlock();
// add a fourth datanode to the cluster
cluster.StartDataNodes(Conf, 1, true, null, NewRacks);
cluster.WaitActive();
DatanodeInfo[] datanodes = client.DatanodeReport(HdfsConstants.DatanodeReportType
.All);
// find out the new node
DatanodeInfo newNode = null;
foreach (DatanodeInfo node in datanodes)
{
bool isNewNode = true;
foreach (DatanodeInfo oldNode in oldNodes)
{
if (node.Equals(oldNode))
{
isNewNode = false;
break;
}
}
if (isNewNode)
{
newNode = node;
break;
}
}
NUnit.Framework.Assert.IsTrue(newNode != null);
DatanodeInfo source = null;
AList <DatanodeInfo> proxies = new AList <DatanodeInfo>(2);
foreach (DatanodeInfo node_1 in datanodes)
{
if (node_1 != newNode)
{
if (node_1.GetNetworkLocation().Equals(newNode.GetNetworkLocation()))
{
source = node_1;
}
else
{
proxies.AddItem(node_1);
}
}
}
//current state: the newNode is on RACK2, and "source" is the other dn on RACK2.
//the two datanodes on RACK0 and RACK1 are in "proxies".
//"source" and both "proxies" all contain the block, while newNode doesn't yet.
NUnit.Framework.Assert.IsTrue(source != null && proxies.Count == 2);
// start to replace the block
// case 1: proxySource does not contain the block
Log.Info("Testcase 1: Proxy " + newNode + " does not contain the block " + b);
NUnit.Framework.Assert.IsFalse(ReplaceBlock(b, source, newNode, proxies[0]));
// case 2: destination already contains the block
Log.Info("Testcase 2: Destination " + proxies[1] + " contains the block " + b);
NUnit.Framework.Assert.IsFalse(ReplaceBlock(b, source, proxies[0], proxies[1]));
// case 3: correct case
Log.Info("Testcase 3: Source=" + source + " Proxy=" + proxies[0] + " Destination="
+ newNode);
NUnit.Framework.Assert.IsTrue(ReplaceBlock(b, source, proxies[0], newNode));
// after cluster has time to resolve the over-replication,
// block locations should contain two proxies and newNode
// but not source
CheckBlocks(new DatanodeInfo[] { newNode, proxies[0], proxies[1] }, fileName.ToString
(), DefaultBlockSize, ReplicationFactor, client);
// case 4: proxies.get(0) is not a valid del hint
// expect either source or newNode replica to be deleted instead
Log.Info("Testcase 4: invalid del hint " + proxies[0]);
NUnit.Framework.Assert.IsTrue(ReplaceBlock(b, proxies[0], proxies[1], source));
// after cluster has time to resolve the over-replication,
// block locations should contain two proxies,
// and either source or newNode, but not both.
CheckBlocks(Sharpen.Collections.ToArray(proxies, new DatanodeInfo[proxies.Count])
, fileName.ToString(), DefaultBlockSize, ReplicationFactor, client);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestProcesOverReplicateBlock()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, 100L);
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L);
conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString
(2));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
Path fileName = new Path("/foo1");
DFSTestUtil.CreateFile(fs, fileName, 2, (short)3, 0L);
DFSTestUtil.WaitReplication(fs, fileName, (short)3);
// corrupt the block on datanode 0
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName);
NUnit.Framework.Assert.IsTrue(cluster.CorruptReplica(0, block));
MiniDFSCluster.DataNodeProperties dnProps = cluster.StopDataNode(0);
// remove block scanner log to trigger block scanning
FilePath scanCursor = new FilePath(new FilePath(MiniDFSCluster.GetFinalizedDir(cluster
.GetInstanceStorageDir(0, 0), cluster.GetNamesystem().GetBlockPoolId()).GetParent
()).GetParent(), "scanner.cursor");
//wait for one minute for deletion to succeed;
for (int i = 0; !scanCursor.Delete(); i++)
{
NUnit.Framework.Assert.IsTrue("Could not delete " + scanCursor.GetAbsolutePath()
+ " in one minute", i < 60);
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
}
// restart the datanode so the corrupt replica will be detected
cluster.RestartDataNode(dnProps);
DFSTestUtil.WaitReplication(fs, fileName, (short)2);
string blockPoolId = cluster.GetNamesystem().GetBlockPoolId();
DatanodeID corruptDataNode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[2], blockPoolId);
FSNamesystem namesystem = cluster.GetNamesystem();
BlockManager bm = namesystem.GetBlockManager();
HeartbeatManager hm = bm.GetDatanodeManager().GetHeartbeatManager();
try
{
namesystem.WriteLock();
lock (hm)
{
// set live datanode's remaining space to be 0
// so they will be chosen to be deleted when over-replication occurs
string corruptMachineName = corruptDataNode.GetXferAddr();
foreach (DatanodeDescriptor datanode in hm.GetDatanodes())
{
if (!corruptMachineName.Equals(datanode.GetXferAddr()))
{
datanode.GetStorageInfos()[0].SetUtilizationForTesting(100L, 100L, 0, 100L);
datanode.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(datanode
), 0L, 0L, 0, 0, null);
}
}
// decrease the replication factor to 1;
NameNodeAdapter.SetReplication(namesystem, fileName.ToString(), (short)1);
// corrupt one won't be chosen to be excess one
// without 4910 the number of live replicas would be 0: block gets lost
NUnit.Framework.Assert.AreEqual(1, bm.CountNodes(block.GetLocalBlock()).LiveReplicas
());
}
}
finally
{
namesystem.WriteUnlock();
}
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Test that we can zero-copy read cached data even without disabling
/// checksums.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestZeroCopyReadOfCachedData()
{
BlockReaderTestUtil.EnableShortCircuitShmTracing();
BlockReaderTestUtil.EnableBlockReaderFactoryTracing();
BlockReaderTestUtil.EnableHdfsCachingTracing();
int TestFileLength = BlockSize;
Path TestPath = new Path("/a");
int RandomSeed = 23453;
HdfsConfiguration conf = InitZeroCopyTest();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
string Context = "testZeroCopyReadOfCachedData";
conf.Set(DFSConfigKeys.DfsClientContext, Context);
conf.SetLong(DFSConfigKeys.DfsDatanodeMaxLockedMemoryKey, DFSTestUtil.RoundUpToMultiple
(TestFileLength, (int)NativeIO.POSIX.GetCacheManipulator().GetOperatingSystemPageSize
()));
MiniDFSCluster cluster = null;
ByteBuffer result = null;
ByteBuffer result2 = null;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
FsDatasetSpi <object> fsd = cluster.GetDataNodes()[0].GetFSDataset();
DistributedFileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed);
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
byte[] original = DFSTestUtil.CalculateFileContentsFromSeed(RandomSeed, TestFileLength
);
// Prior to caching, the file can't be read via zero-copy
FSDataInputStream fsIn = fs.Open(TestPath);
try
{
result = fsIn.Read(null, TestFileLength / 2, EnumSet.NoneOf <ReadOption>());
NUnit.Framework.Assert.Fail("expected UnsupportedOperationException");
}
catch (NotSupportedException)
{
}
// expected
// Cache the file
fs.AddCachePool(new CachePoolInfo("pool1"));
long directiveId = fs.AddCacheDirective(new CacheDirectiveInfo.Builder().SetPath(
TestPath).SetReplication((short)1).SetPool("pool1").Build());
int numBlocks = (int)Math.Ceil((double)TestFileLength / BlockSize);
DFSTestUtil.VerifyExpectedCacheUsage(DFSTestUtil.RoundUpToMultiple(TestFileLength
, BlockSize), numBlocks, cluster.GetDataNodes()[0].GetFSDataset());
try
{
result = fsIn.Read(null, TestFileLength, EnumSet.NoneOf <ReadOption>());
}
catch (NotSupportedException)
{
NUnit.Framework.Assert.Fail("expected to be able to read cached file via zero-copy"
);
}
Assert.AssertArrayEquals(Arrays.CopyOfRange(original, 0, BlockSize), ByteBufferToArray
(result));
// Test that files opened after the cache operation has finished
// still get the benefits of zero-copy (regression test for HDFS-6086)
FSDataInputStream fsIn2 = fs.Open(TestPath);
try
{
result2 = fsIn2.Read(null, TestFileLength, EnumSet.NoneOf <ReadOption>());
}
catch (NotSupportedException)
{
NUnit.Framework.Assert.Fail("expected to be able to read cached file via zero-copy"
);
}
Assert.AssertArrayEquals(Arrays.CopyOfRange(original, 0, BlockSize), ByteBufferToArray
(result2));
fsIn2.ReleaseBuffer(result2);
fsIn2.Close();
// check that the replica is anchored
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fs, TestPath);
ShortCircuitCache cache = ClientContext.Get(Context, new DFSClient.Conf(conf)).GetShortCircuitCache
();
WaitForReplicaAnchorStatus(cache, firstBlock, true, true, 1);
// Uncache the replica
fs.RemoveCacheDirective(directiveId);
WaitForReplicaAnchorStatus(cache, firstBlock, false, true, 1);
fsIn.ReleaseBuffer(result);
WaitForReplicaAnchorStatus(cache, firstBlock, false, false, 1);
DFSTestUtil.VerifyExpectedCacheUsage(0, 0, fsd);
fsIn.Close();
fs.Close();
cluster.Shutdown();
}
