/// <summary>
/// Test that the NN initializes its under-replicated blocks queue
/// before it is ready to exit safemode (HDFS-1476)
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestInitializeReplQueuesEarly()
{
Log.Info("Starting testInitializeReplQueuesEarly");
// Spray the blocks around the cluster when we add DNs instead of
// concentrating all blocks on the first node.
BlockManagerTestUtil.SetWritingPrefersLocalNode(cluster.GetNamesystem().GetBlockManager
(), false);
cluster.StartDataNodes(conf, 2, true, HdfsServerConstants.StartupOption.Regular,
null);
cluster.WaitActive();
Log.Info("Creating files");
DFSTestUtil.CreateFile(fs, TestPath, 15 * BlockSize, (short)1, 1L);
Log.Info("Stopping all DataNodes");
IList <MiniDFSCluster.DataNodeProperties> dnprops = Lists.NewLinkedList();
dnprops.AddItem(cluster.StopDataNode(0));
dnprops.AddItem(cluster.StopDataNode(0));
dnprops.AddItem(cluster.StopDataNode(0));
cluster.GetConfiguration(0).SetFloat(DFSConfigKeys.DfsNamenodeReplQueueThresholdPctKey
, 1f / 15f);
Log.Info("Restarting NameNode");
cluster.RestartNameNode();
NameNode nn = cluster.GetNameNode();
string status = nn.GetNamesystem().GetSafemode();
NUnit.Framework.Assert.AreEqual("Safe mode is ON. The reported blocks 0 needs additional "
+ "15 blocks to reach the threshold 0.9990 of total blocks 15." + Newline + "The number of live datanodes 0 has reached the minimum number 0. "
+ "Safe mode will be turned off automatically once the thresholds " + "have been reached."
, status);
NUnit.Framework.Assert.IsFalse("Mis-replicated block queues should not be initialized "
+ "until threshold is crossed", NameNodeAdapter.SafeModeInitializedReplQueues(nn
));
Log.Info("Restarting one DataNode");
cluster.RestartDataNode(dnprops.Remove(0));
// Wait for block reports from all attached storages of
// the restarted DN to come in.
GenericTestUtils.WaitFor(new _Supplier_214(this), 10, 10000);
int safe = NameNodeAdapter.GetSafeModeSafeBlocks(nn);
NUnit.Framework.Assert.IsTrue("Expected first block report to make some blocks safe."
, safe > 0);
NUnit.Framework.Assert.IsTrue("Did not expect first block report to make all blocks safe."
, safe < 15);
NUnit.Framework.Assert.IsTrue(NameNodeAdapter.SafeModeInitializedReplQueues(nn));
// Ensure that UnderReplicatedBlocks goes up to 15 - safe. Misreplicated
// blocks are processed asynchronously so this may take a few seconds.
// Failure here will manifest as a test timeout.
BlockManagerTestUtil.UpdateState(nn.GetNamesystem().GetBlockManager());
long underReplicatedBlocks = nn.GetNamesystem().GetUnderReplicatedBlocks();
while (underReplicatedBlocks != (15 - safe))
{
Log.Info("UnderReplicatedBlocks expected=" + (15 - safe) + ", actual=" + underReplicatedBlocks
);
Sharpen.Thread.Sleep(100);
BlockManagerTestUtil.UpdateState(nn.GetNamesystem().GetBlockManager());
underReplicatedBlocks = nn.GetNamesystem().GetUnderReplicatedBlocks();
}
cluster.RestartDataNodes();
}
public virtual void EnsureSerialNumbersNeverOverlap()
{
BlockTokenSecretManager btsm1 = cluster.GetNamesystem(0).GetBlockManager().GetBlockTokenSecretManager
();
BlockTokenSecretManager btsm2 = cluster.GetNamesystem(1).GetBlockManager().GetBlockTokenSecretManager
();
btsm1.SetSerialNo(0);
btsm2.SetSerialNo(0);
NUnit.Framework.Assert.IsFalse(btsm1.GetSerialNoForTesting() == btsm2.GetSerialNoForTesting
());
btsm1.SetSerialNo(int.MaxValue);
btsm2.SetSerialNo(int.MaxValue);
NUnit.Framework.Assert.IsFalse(btsm1.GetSerialNoForTesting() == btsm2.GetSerialNoForTesting
());
btsm1.SetSerialNo(int.MinValue);
btsm2.SetSerialNo(int.MinValue);
NUnit.Framework.Assert.IsFalse(btsm1.GetSerialNoForTesting() == btsm2.GetSerialNoForTesting
());
btsm1.SetSerialNo(int.MaxValue / 2);
btsm2.SetSerialNo(int.MaxValue / 2);
NUnit.Framework.Assert.IsFalse(btsm1.GetSerialNoForTesting() == btsm2.GetSerialNoForTesting
());
btsm1.SetSerialNo(int.MinValue / 2);
btsm2.SetSerialNo(int.MinValue / 2);
NUnit.Framework.Assert.IsFalse(btsm1.GetSerialNoForTesting() == btsm2.GetSerialNoForTesting
());
}
private static void AssertCTimesEqual(MiniDFSCluster cluster)
{
long nn1CTime = cluster.GetNamesystem(0).GetFSImage().GetStorage().GetCTime();
long nn2CTime = cluster.GetNamesystem(1).GetFSImage().GetStorage().GetCTime();
NUnit.Framework.Assert.AreEqual(nn1CTime, nn2CTime);
}
public virtual void TestConfigureMinValidVolumes()
{
Assume.AssumeTrue(!Runtime.GetProperty("os.name").StartsWith("Windows"));
// Bring up two additional datanodes that need both of their volumes
// functioning in order to stay up.
conf.SetInt(DFSConfigKeys.DfsDatanodeFailedVolumesToleratedKey, 0);
cluster.StartDataNodes(conf, 2, true, null, null);
cluster.WaitActive();
DatanodeManager dm = cluster.GetNamesystem().GetBlockManager().GetDatanodeManager
();
long origCapacity = DFSTestUtil.GetLiveDatanodeCapacity(dm);
long dnCapacity = DFSTestUtil.GetDatanodeCapacity(dm, 0);
// Fail a volume on the 2nd DN
FilePath dn2Vol1 = new FilePath(dataDir, "data" + (2 * 1 + 1));
DataNodeTestUtils.InjectDataDirFailure(dn2Vol1);
// Should only get two replicas (the first DN and the 3rd)
Path file1 = new Path("/test1");
DFSTestUtil.CreateFile(fs, file1, 1024, (short)3, 1L);
DFSTestUtil.WaitReplication(fs, file1, (short)2);
// Check that this single failure caused a DN to die.
DFSTestUtil.WaitForDatanodeStatus(dm, 2, 1, 0, origCapacity - (1 * dnCapacity), WaitForHeartbeats
);
// If we restore the volume we should still only be able to get
// two replicas since the DN is still considered dead.
DataNodeTestUtils.RestoreDataDirFromFailure(dn2Vol1);
Path file2 = new Path("/test2");
DFSTestUtil.CreateFile(fs, file2, 1024, (short)3, 1L);
DFSTestUtil.WaitReplication(fs, file2, (short)2);
}
public virtual void TestRetryCacheRebuild()
{
DFSTestUtil.RunOperations(cluster, filesystem, conf, BlockSize, 0);
FSNamesystem namesystem = cluster.GetNamesystem();
LightWeightCache <RetryCache.CacheEntry, RetryCache.CacheEntry> cacheSet = (LightWeightCache
<RetryCache.CacheEntry, RetryCache.CacheEntry>)namesystem.GetRetryCache().GetCacheSet
();
NUnit.Framework.Assert.AreEqual(25, cacheSet.Size());
IDictionary <RetryCache.CacheEntry, RetryCache.CacheEntry> oldEntries = new Dictionary
<RetryCache.CacheEntry, RetryCache.CacheEntry>();
IEnumerator <RetryCache.CacheEntry> iter = cacheSet.GetEnumerator();
while (iter.HasNext())
{
RetryCache.CacheEntry entry = iter.Next();
oldEntries[entry] = entry;
}
// restart NameNode
cluster.RestartNameNode();
cluster.WaitActive();
namesystem = cluster.GetNamesystem();
// check retry cache
NUnit.Framework.Assert.IsTrue(namesystem.HasRetryCache());
cacheSet = (LightWeightCache <RetryCache.CacheEntry, RetryCache.CacheEntry>)namesystem
.GetRetryCache().GetCacheSet();
NUnit.Framework.Assert.AreEqual(25, cacheSet.Size());
iter = cacheSet.GetEnumerator();
while (iter.HasNext())
{
RetryCache.CacheEntry entry = iter.Next();
NUnit.Framework.Assert.IsTrue(oldEntries.Contains(entry));
}
}
public virtual void TestCorruptBlock()
{
// Create a file with single block with two replicas
Path file = GetTestPath("testCorruptBlock");
CreateFile(file, 100, (short)2);
// Corrupt first replica of the block
LocatedBlock block = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), file
.ToString(), 0, 1).Get(0);
cluster.GetNamesystem().WriteLock();
try
{
bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[0], "STORAGE_ID"
, "TEST");
}
finally
{
cluster.GetNamesystem().WriteUnlock();
}
UpdateMetrics();
MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(NsMetrics);
MetricsAsserts.AssertGauge("CorruptBlocks", 1L, rb);
MetricsAsserts.AssertGauge("PendingReplicationBlocks", 1L, rb);
MetricsAsserts.AssertGauge("ScheduledReplicationBlocks", 1L, rb);
fs.Delete(file, true);
rb = WaitForDnMetricValue(NsMetrics, "CorruptBlocks", 0L);
MetricsAsserts.AssertGauge("PendingReplicationBlocks", 0L, rb);
MetricsAsserts.AssertGauge("ScheduledReplicationBlocks", 0L, rb);
}
public virtual void TestSnapshottableDirs()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 0, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("AllowSnapshotOps", 0L, MetricsAsserts.GetMetrics(NnMetrics
));
MetricsAsserts.AssertCounter("DisallowSnapshotOps", 0L, MetricsAsserts.GetMetrics
(NnMetrics));
// Allow snapshots for directories, and check the metrics
hdfs.AllowSnapshot(sub1);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 1, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("AllowSnapshotOps", 1L, MetricsAsserts.GetMetrics(NnMetrics
));
Path sub2 = new Path(dir, "sub2");
Path file = new Path(sub2, "file");
DFSTestUtil.CreateFile(hdfs, file, 1024, Replication, seed);
hdfs.AllowSnapshot(sub2);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 2, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("AllowSnapshotOps", 2L, MetricsAsserts.GetMetrics(NnMetrics
));
Path subsub1 = new Path(sub1, "sub1sub1");
Path subfile = new Path(subsub1, "file");
DFSTestUtil.CreateFile(hdfs, subfile, 1024, Replication, seed);
hdfs.AllowSnapshot(subsub1);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 3, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("AllowSnapshotOps", 3L, MetricsAsserts.GetMetrics(NnMetrics
));
// Set an already snapshottable directory to snapshottable, should not
// change the metrics
hdfs.AllowSnapshot(sub1);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 3, MetricsAsserts.GetMetrics
(NsMetrics));
// But the number of allowSnapshot operations still increases
MetricsAsserts.AssertCounter("AllowSnapshotOps", 4L, MetricsAsserts.GetMetrics(NnMetrics
));
// Disallow the snapshot for snapshottable directories, then check the
// metrics again
hdfs.DisallowSnapshot(sub1);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 2, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsAsserts.AssertCounter("DisallowSnapshotOps", 1L, MetricsAsserts.GetMetrics
(NnMetrics));
// delete subsub1, snapshottable directories should be 1
hdfs.Delete(subsub1, true);
MetricsAsserts.AssertGauge("SnapshottableDirectories", 1, MetricsAsserts.GetMetrics
(NsMetrics));
// list all the snapshottable directories
SnapshottableDirectoryStatus[] status = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, status.Length);
MetricsAsserts.AssertCounter("ListSnapshottableDirOps", 1L, MetricsAsserts.GetMetrics
(NnMetrics));
}
private static void LowerKeyUpdateIntervalAndClearKeys(MiniDFSCluster cluster)
{
LowerKeyUpdateIntervalAndClearKeys(cluster.GetNamesystem(0));
LowerKeyUpdateIntervalAndClearKeys(cluster.GetNamesystem(1));
foreach (DataNode dn in cluster.GetDataNodes())
{
dn.ClearAllBlockSecretKeys();
}
}
public virtual void TestAddBlock()
{
DistributedFileSystem fs = cluster.GetFileSystem();
Path file1 = new Path("/file1");
Path file2 = new Path("/file2");
Path file3 = new Path("/file3");
Path file4 = new Path("/file4");
DFSTestUtil.CreateFile(fs, file1, Blocksize - 1, Replication, 0L);
DFSTestUtil.CreateFile(fs, file2, Blocksize, Replication, 0L);
DFSTestUtil.CreateFile(fs, file3, Blocksize * 2 - 1, Replication, 0L);
DFSTestUtil.CreateFile(fs, file4, Blocksize * 2, Replication, 0L);
// restart NameNode
cluster.RestartNameNode(true);
FSDirectory fsdir = cluster.GetNamesystem().GetFSDirectory();
// check file1
INodeFile file1Node = fsdir.GetINode4Write(file1.ToString()).AsFile();
BlockInfoContiguous[] file1Blocks = file1Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, file1Blocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize - 1, file1Blocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file1Blocks
[0].GetBlockUCState());
// check file2
INodeFile file2Node = fsdir.GetINode4Write(file2.ToString()).AsFile();
BlockInfoContiguous[] file2Blocks = file2Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, file2Blocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, file2Blocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file2Blocks
[0].GetBlockUCState());
// check file3
INodeFile file3Node = fsdir.GetINode4Write(file3.ToString()).AsFile();
BlockInfoContiguous[] file3Blocks = file3Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(2, file3Blocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, file3Blocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file3Blocks
[0].GetBlockUCState());
NUnit.Framework.Assert.AreEqual(Blocksize - 1, file3Blocks[1].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file3Blocks
[1].GetBlockUCState());
// check file4
INodeFile file4Node = fsdir.GetINode4Write(file4.ToString()).AsFile();
BlockInfoContiguous[] file4Blocks = file4Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(2, file4Blocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, file4Blocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file4Blocks
[0].GetBlockUCState());
NUnit.Framework.Assert.AreEqual(Blocksize, file4Blocks[1].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file4Blocks
[1].GetBlockUCState());
}
/// <exception cref="System.IO.IOException"/>
private LocatedBlocks CreateFileGetBlocks(string filenamePrefix)
{
Path filePath = new Path("/" + filenamePrefix + ".dat");
// Write out a file with a few blocks, get block locations.
DFSTestUtil.CreateFile(fs, filePath, BlockSize, BlockSize * NumBlocks, BlockSize,
NumDatanodes, seed);
// Get the block list for the file with the block locations.
LocatedBlocks blocks = client.GetLocatedBlocks(filePath.ToString(), 0, BlockSize
* NumBlocks);
Assert.AssertThat(cluster.GetNamesystem().GetUnderReplicatedBlocks(), IS.Is(0L));
return(blocks);
}
/// <summary>
/// Run file operations to create edits for all op codes
/// to be tested.
/// </summary>
/// <remarks>
/// Run file operations to create edits for all op codes
/// to be tested.
/// the following op codes are deprecated and therefore not tested:
/// OP_DATANODE_ADD ( 5)
/// OP_DATANODE_REMOVE ( 6)
/// OP_SET_NS_QUOTA (11)
/// OP_CLEAR_NS_QUOTA (12)
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private CheckpointSignature RunOperations()
{
Log.Info("Creating edits by performing fs operations");
// no check, if it's not it throws an exception which is what we want
DistributedFileSystem dfs = cluster.GetFileSystem();
DFSTestUtil.RunOperations(cluster, dfs, cluster.GetConfiguration(0), dfs.GetDefaultBlockSize
(), 0);
// OP_ROLLING_UPGRADE_START
cluster.GetNamesystem().GetEditLog().LogStartRollingUpgrade(Time.Now());
// OP_ROLLING_UPGRADE_FINALIZE
cluster.GetNamesystem().GetEditLog().LogFinalizeRollingUpgrade(Time.Now());
// Force a roll so we get an OP_END_LOG_SEGMENT txn
return(cluster.GetNameNodeRpc().RollEditLog());
}
/// <exception cref="System.Exception"/>
public virtual void TestQuery()
{
Configuration conf = new Configuration();
MiniQJMHACluster cluster = null;
try
{
cluster = new MiniQJMHACluster.Builder(conf).Build();
MiniDFSCluster dfsCluster = cluster.GetDfsCluster();
dfsCluster.WaitActive();
dfsCluster.TransitionToActive(0);
DistributedFileSystem dfs = dfsCluster.GetFileSystem(0);
dfsCluster.ShutdownNameNode(1);
// start rolling upgrade
RollingUpgradeInfo info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare
);
NUnit.Framework.Assert.IsTrue(info.IsStarted());
info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Query);
NUnit.Framework.Assert.IsFalse(info.CreatedRollbackImages());
dfsCluster.RestartNameNode(1);
QueryForPreparation(dfs);
// The NN should have a copy of the fsimage in case of rollbacks.
NUnit.Framework.Assert.IsTrue(dfsCluster.GetNamesystem(0).GetFSImage().HasRollbackFSImage
());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public static void SetUp()
{
conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsNamenodeReplicationConsiderloadKey, false);
// Set up the hosts/exclude files.
localFileSys = FileSystem.GetLocal(conf);
Path workingDir = localFileSys.GetWorkingDirectory();
dir = new Path(workingDir, "build/test/data/work-dir/decommission");
NUnit.Framework.Assert.IsTrue(localFileSys.Mkdirs(dir));
excludeFile = new Path(dir, "exclude");
conf.Set(DFSConfigKeys.DfsHostsExclude, excludeFile.ToUri().GetPath());
Path includeFile = new Path(dir, "include");
conf.Set(DFSConfigKeys.DfsHosts, includeFile.ToUri().GetPath());
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1000);
conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, 4);
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1000);
conf.SetInt(DFSConfigKeys.DfsNamenodeDecommissionIntervalKey, 1);
conf.SetLong(DFSConfigKeys.DfsDatanodeBalanceBandwidthpersecKey, 1);
WriteConfigFile(localFileSys, excludeFile, null);
WriteConfigFile(localFileSys, includeFile, null);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes).Build();
cluster.WaitActive();
fileSys = cluster.GetFileSystem();
cluster.GetNamesystem().GetBlockManager().GetDatanodeManager().SetHeartbeatExpireInterval
(3000);
Logger.GetLogger(typeof(DecommissionManager)).SetLevel(Level.Debug);
}
public virtual void TestBlocksScheduledCounter()
{
cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
//open a file an write a few bytes:
FSDataOutputStream @out = fs.Create(new Path("/testBlockScheduledCounter"));
for (int i = 0; i < 1024; i++)
{
@out.Write(i);
}
// flush to make sure a block is allocated.
@out.Hflush();
AList <DatanodeDescriptor> dnList = new AList <DatanodeDescriptor>();
DatanodeManager dm = cluster.GetNamesystem().GetBlockManager().GetDatanodeManager
();
dm.FetchDatanodes(dnList, dnList, false);
DatanodeDescriptor dn = dnList[0];
NUnit.Framework.Assert.AreEqual(1, dn.GetBlocksScheduled());
// close the file and the counter should go to zero.
@out.Close();
NUnit.Framework.Assert.AreEqual(0, dn.GetBlocksScheduled());
}
/// <exception cref="System.IO.IOException"/>
private void CorruptBlock(MiniDFSCluster cluster, FileSystem fs, Path fileName, int
dnIndex, ExtendedBlock block)
{
// corrupt the block on datanode dnIndex
// the indexes change once the nodes are restarted.
// But the datadirectory will not change
NUnit.Framework.Assert.IsTrue(cluster.CorruptReplica(dnIndex, block));
MiniDFSCluster.DataNodeProperties dnProps = cluster.StopDataNode(0);
// Each datanode has multiple data dirs, check each
for (int dirIndex = 0; dirIndex < 2; dirIndex++)
{
string bpid = cluster.GetNamesystem().GetBlockPoolId();
FilePath storageDir = cluster.GetStorageDir(dnIndex, dirIndex);
FilePath dataDir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid);
FilePath scanLogFile = new FilePath(dataDir, "dncp_block_verification.log.curr");
if (scanLogFile.Exists())
{
// wait for one minute for deletion to succeed;
for (int i = 0; !scanLogFile.Delete(); i++)
{
NUnit.Framework.Assert.IsTrue("Could not delete log file in one minute", i < 60);
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
}
}
}
// restart the detained so the corrupt replica will be detected
cluster.RestartDataNode(dnProps);
}
/// <exception cref="System.IO.IOException"/>
private void TestRbwReplicas(MiniDFSCluster cluster, bool isCorrupt)
{
FSDataOutputStream @out = null;
FileSystem fs = cluster.GetFileSystem();
Path src = new Path("/test.txt");
try
{
int fileLen = 515;
// create some rbw replicas on disk
byte[] writeBuf = new byte[fileLen];
new Random().NextBytes(writeBuf);
@out = fs.Create(src);
@out.Write(writeBuf);
@out.Hflush();
DataNode dn = cluster.GetDataNodes()[0];
foreach (FsVolumeSpi v in Dataset(dn).GetVolumes())
{
FsVolumeImpl volume = (FsVolumeImpl)v;
FilePath currentDir = volume.GetCurrentDir().GetParentFile().GetParentFile();
FilePath rbwDir = new FilePath(currentDir, "rbw");
foreach (FilePath file in rbwDir.ListFiles())
{
if (isCorrupt && Block.IsBlockFilename(file))
{
new RandomAccessFile(file, "rw").SetLength(fileLen - 1);
}
}
}
// corrupt
cluster.RestartDataNodes();
cluster.WaitActive();
dn = cluster.GetDataNodes()[0];
// check volumeMap: one rwr replica
string bpid = cluster.GetNamesystem().GetBlockPoolId();
ReplicaMap replicas = Dataset(dn).volumeMap;
NUnit.Framework.Assert.AreEqual(1, replicas.Size(bpid));
ReplicaInfo replica = replicas.Replicas(bpid).GetEnumerator().Next();
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.ReplicaState.Rwr, replica.GetState
());
if (isCorrupt)
{
NUnit.Framework.Assert.AreEqual((fileLen - 1) / 512 * 512, replica.GetNumBytes());
}
else
{
NUnit.Framework.Assert.AreEqual(fileLen, replica.GetNumBytes());
}
Dataset(dn).Invalidate(bpid, new Block[] { replica });
}
finally
{
IOUtils.CloseStream(@out);
if (fs.Exists(src))
{
fs.Delete(src, false);
}
fs.Close();
}
}
/// <summary>Initializes the cluster.</summary>
/// <param name="numDataNodes">number of datanodes</param>
/// <param name="storagesPerDatanode">number of storage locations on each datanode</param>
/// <param name="failedVolumesTolerated">number of acceptable volume failures</param>
/// <exception cref="System.Exception">if there is any failure</exception>
private void InitCluster(int numDataNodes, int storagesPerDatanode, int failedVolumesTolerated
)
{
conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 512L);
/*
* Lower the DN heartbeat, DF rate, and recheck interval to one second
* so state about failures and datanode death propagates faster.
*/
conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
conf.SetInt(DFSConfigKeys.DfsDfIntervalKey, 1000);
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1000);
conf.SetInt(DFSConfigKeys.DfsDatanodeFailedVolumesToleratedKey, failedVolumesTolerated
);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).StoragesPerDatanode
(storagesPerDatanode).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
dataDir = cluster.GetDataDirectory();
long dnCapacity = DFSTestUtil.GetDatanodeCapacity(cluster.GetNamesystem().GetBlockManager
().GetDatanodeManager(), 0);
volumeCapacity = dnCapacity / cluster.GetStoragesPerDatanode();
}
public virtual void Pipeline_01()
{
string MethodName = GenericTestUtils.GetMethodName();
if (Log.IsDebugEnabled())
{
Log.Debug("Running " + MethodName);
}
Path filePath = new Path("/" + MethodName + ".dat");
DFSTestUtil.CreateFile(fs, filePath, FileSize, ReplFactor, rand.NextLong());
if (Log.IsDebugEnabled())
{
Log.Debug("Invoking append but doing nothing otherwise...");
}
FSDataOutputStream ofs = fs.Append(filePath);
ofs.WriteBytes("Some more stuff to write");
((DFSOutputStream)ofs.GetWrappedStream()).Hflush();
IList <LocatedBlock> lb = cluster.GetNameNodeRpc().GetBlockLocations(filePath.ToString
(), FileSize - 1, FileSize).GetLocatedBlocks();
string bpid = cluster.GetNamesystem().GetBlockPoolId();
foreach (DataNode dn in cluster.GetDataNodes())
{
Replica r = DataNodeTestUtils.FetchReplicaInfo(dn, bpid, lb[0].GetBlock().GetBlockId
());
NUnit.Framework.Assert.IsTrue("Replica on DN " + dn + " shouldn't be null", r !=
null);
NUnit.Framework.Assert.AreEqual("Should be RBW replica on " + dn + " after sequence of calls append()/write()/hflush()"
, HdfsServerConstants.ReplicaState.Rbw, r.GetState());
}
ofs.Close();
}
/// <exception cref="System.Exception"/>
internal TestContext(Configuration conf, int numNameServices)
{
this.numNameServices = numNameServices;
MiniDFSCluster.Builder bld = new MiniDFSCluster.Builder(conf).NumDataNodes(1).StoragesPerDatanode
(1);
if (numNameServices > 1)
{
bld.NnTopology(MiniDFSNNTopology.SimpleFederatedTopology(numNameServices));
}
cluster = bld.Build();
cluster.WaitActive();
dfs = new DistributedFileSystem[numNameServices];
for (int i = 0; i < numNameServices; i++)
{
dfs[i] = cluster.GetFileSystem(i);
}
bpids = new string[numNameServices];
for (int i_1 = 0; i_1 < numNameServices; i_1++)
{
bpids[i_1] = cluster.GetNamesystem(i_1).GetBlockPoolId();
}
datanode = cluster.GetDataNodes()[0];
blockScanner = datanode.GetBlockScanner();
for (int i_2 = 0; i_2 < numNameServices; i_2++)
{
dfs[i_2].Mkdirs(new Path("/test"));
}
data = datanode.GetFSDataset();
volumes = data.GetVolumes();
}
/// <summary>Pull out one of the JournalAndStream objects from the edit log.</summary>
private JournalSet.JournalAndStream GetJournalAndStream(int index)
{
FSImage fsimage = cluster.GetNamesystem().GetFSImage();
FSEditLog editLog = fsimage.GetEditLog();
return(editLog.GetJournals()[index]);
}
public virtual void SetUp()
{
cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(DatanodeCount).Build();
cluster.WaitActive();
namesystem = cluster.GetNamesystem();
bm = namesystem.GetBlockManager();
fs = cluster.GetFileSystem();
}
public virtual void SetUp()
{
conf = new Configuration();
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(Replication).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
}
/// <exception cref="System.IO.IOException"/>
public virtual void StartUpCluster(long splitThreshold)
{
conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockreportSplitThresholdKey, splitThreshold);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(ReplFactor).Build();
fs = cluster.GetFileSystem();
bpid = cluster.GetNamesystem().GetBlockPoolId();
}
public virtual void StartUpCluster()
{
conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDatanodes).Build();
fs = cluster.GetFileSystem();
client = fs.GetClient();
bpid = cluster.GetNamesystem().GetBlockPoolId();
}
public virtual void StartUpCluster()
{
conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(ReplFactor).StorageTypes(
new StorageType[] { storageType, storageType }).Build();
fs = cluster.GetFileSystem();
bpid = cluster.GetNamesystem().GetBlockPoolId();
}
public virtual void TestRetryAddBlockWhileInChooseTarget()
{
string src = "/testRetryAddBlockWhileInChooseTarget";
FSNamesystem ns = cluster.GetNamesystem();
NamenodeProtocols nn = cluster.GetNameNodeRpc();
// create file
nn.Create(src, FsPermission.GetFileDefault(), "clientName", new EnumSetWritable <CreateFlag
>(EnumSet.Of(CreateFlag.Create)), true, (short)3, 1024, null);
// start first addBlock()
Log.Info("Starting first addBlock for " + src);
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
DatanodeStorageInfo[] targets = ns.GetNewBlockTargets(src, INodeId.GrandfatherInodeId
, "clientName", null, null, null, onRetryBlock);
NUnit.Framework.Assert.IsNotNull("Targets must be generated", targets);
// run second addBlock()
Log.Info("Starting second addBlock for " + src);
nn.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId, null);
NUnit.Framework.Assert.IsTrue("Penultimate block must be complete", CheckFileProgress
(src, false));
LocatedBlocks lbs = nn.GetBlockLocations(src, 0, long.MaxValue);
NUnit.Framework.Assert.AreEqual("Must be one block", 1, lbs.GetLocatedBlocks().Count
);
LocatedBlock lb2 = lbs.Get(0);
NUnit.Framework.Assert.AreEqual("Wrong replication", Replication, lb2.GetLocations
().Length);
// continue first addBlock()
LocatedBlock newBlock = ns.StoreAllocatedBlock(src, INodeId.GrandfatherInodeId, "clientName"
, null, targets);
NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb2.GetBlock(), newBlock.
GetBlock());
// check locations
lbs = nn.GetBlockLocations(src, 0, long.MaxValue);
NUnit.Framework.Assert.AreEqual("Must be one block", 1, lbs.GetLocatedBlocks().Count
);
LocatedBlock lb1 = lbs.Get(0);
NUnit.Framework.Assert.AreEqual("Wrong replication", Replication, lb1.GetLocations
().Length);
NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb1.GetBlock(), lb2.GetBlock
());
}
private IList <IList <int> > GetNumBlocksReport(int namesystemIdx)
{
IList <IList <int> > results = new AList <IList <int> >();
string bpid = cluster.GetNamesystem(namesystemIdx).GetBlockPoolId();
IList <IDictionary <DatanodeStorage, BlockListAsLongs> > blockReports = cluster.GetAllBlockReports
(bpid);
foreach (IDictionary <DatanodeStorage, BlockListAsLongs> datanodeReport in blockReports)
{
IList <int> numBlocksPerDN = new AList <int>();
foreach (BlockListAsLongs blocks in datanodeReport.Values)
{
numBlocksPerDN.AddItem(blocks.GetNumberOfBlocks());
}
results.AddItem(numBlocksPerDN);
}
return(results);
}
public virtual void SetUp()
{
conf = new Configuration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, Blocksize);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(Replication).Build();
cluster.WaitActive();
fsdir = cluster.GetNamesystem().GetFSDirectory();
dfs = cluster.GetFileSystem();
}
public virtual void SetUp()
{
conf = new Configuration();
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(Numdatanode).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
fsdir = fsn.GetFSDirectory();
}
public virtual void TestDeadDatanode()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 500);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L);
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
// wait for datanode to be marked live
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration reg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes
()[0], poolId);
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), true, 20000);
// Shutdown and wait for datanode to be marked dead
dn.Shutdown();
DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), false, 20000);
DatanodeProtocol dnp = cluster.GetNameNodeRpc();
ReceivedDeletedBlockInfo[] blocks = new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo
(new Block(0), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null) };
StorageReceivedDeletedBlocks[] storageBlocks = new StorageReceivedDeletedBlocks[]
{ new StorageReceivedDeletedBlocks(reg.GetDatanodeUuid(), blocks) };
// Ensure blockReceived call from dead datanode is rejected with IOException
try
{
dnp.BlockReceivedAndDeleted(reg, poolId, storageBlocks);
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure blockReport from dead datanode is rejected with IOException
StorageBlockReport[] report = new StorageBlockReport[] { new StorageBlockReport(new
DatanodeStorage(reg.GetDatanodeUuid()), BlockListAsLongs.Empty) };
try
{
dnp.BlockReport(reg, poolId, report, new BlockReportContext(1, 0, Runtime.NanoTime
()));
NUnit.Framework.Assert.Fail("Expected IOException is not thrown");
}
catch (IOException)
{
}
// Expected
// Ensure heartbeat from dead datanode is rejected with a command
// that asks datanode to register again
StorageReport[] rep = new StorageReport[] { new StorageReport(new DatanodeStorage
(reg.GetDatanodeUuid()), false, 0, 0, 0, 0) };
DatanodeCommand[] cmd = dnp.SendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null).GetCommands
();
NUnit.Framework.Assert.AreEqual(1, cmd.Length);
NUnit.Framework.Assert.AreEqual(cmd[0].GetAction(), RegisterCommand.Register.GetAction
());
}