/// <summary>TC7: Corrupted replicas are present.</summary>
/// <exception cref="System.IO.IOException">an exception might be thrown</exception>
/// <exception cref="System.Exception"/>
private void TestTC7(bool appendToNewBlock)
{
short repl = 2;
Path p = new Path("/TC7/foo" + (appendToNewBlock ? "0" : "1"));
System.Console.Out.WriteLine("p=" + p);
//a. Create file with replication factor of 2. Write half block of data. Close file.
int len1 = (int)(BlockSize / 2);
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, repl, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
DFSTestUtil.WaitReplication(fs, p, repl);
//b. Log into one datanode that has one replica of this block.
// Find the block file on this datanode and truncate it to zero size.
LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(p.ToString()
, 0L, len1);
NUnit.Framework.Assert.AreEqual(1, locatedblocks.LocatedBlockCount());
LocatedBlock lb = locatedblocks.Get(0);
ExtendedBlock blk = lb.GetBlock();
NUnit.Framework.Assert.AreEqual(len1, lb.GetBlockSize());
DatanodeInfo[] datanodeinfos = lb.GetLocations();
NUnit.Framework.Assert.AreEqual(repl, datanodeinfos.Length);
DataNode dn = cluster.GetDataNode(datanodeinfos[0].GetIpcPort());
FilePath f = DataNodeTestUtils.GetBlockFile(dn, blk.GetBlockPoolId(), blk.GetLocalBlock
());
RandomAccessFile raf = new RandomAccessFile(f, "rw");
AppendTestUtil.Log.Info("dn=" + dn + ", blk=" + blk + " (length=" + blk.GetNumBytes
() + ")");
NUnit.Framework.Assert.AreEqual(len1, raf.Length());
raf.SetLength(0);
raf.Close();
//c. Open file in "append mode". Append a new block worth of data. Close file.
int len2 = (int)BlockSize;
{
FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append
, CreateFlag.NewBlock), 4096, null) : fs.Append(p);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
//d. Reopen file and read two blocks worth of data.
AppendTestUtil.Check(fs, p, len1 + len2);
}
/// <summary>Create a file with one block and corrupt some/all of the block replicas.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/>
/// <exception cref="System.IO.FileNotFoundException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.TimeoutException"/>
private void CreateAFileWithCorruptedBlockReplicas(Path filePath, short repl, int
corruptBlockCount)
{
DFSTestUtil.CreateFile(dfs, filePath, BlockSize, repl, 0);
DFSTestUtil.WaitReplication(dfs, filePath, repl);
// Locate the file blocks by asking name node
LocatedBlocks locatedblocks = dfs.dfs.GetNamenode().GetBlockLocations(filePath.ToString
(), 0L, BlockSize);
NUnit.Framework.Assert.AreEqual(repl, locatedblocks.Get(0).GetLocations().Length);
// The file only has one block
LocatedBlock lblock = locatedblocks.Get(0);
DatanodeInfo[] datanodeinfos = lblock.GetLocations();
ExtendedBlock block = lblock.GetBlock();
// corrupt some /all of the block replicas
for (int i = 0; i < corruptBlockCount; i++)
{
DatanodeInfo dninfo = datanodeinfos[i];
DataNode dn = cluster.GetDataNode(dninfo.GetIpcPort());
CorruptBlock(block, dn);
Log.Debug("Corrupted block " + block.GetBlockName() + " on data node " + dninfo);
}
}
/// <summary>Get a DataNode that serves our testBlock.</summary>
public virtual DataNode GetDataNode(LocatedBlock testBlock)
{
DatanodeInfo[] nodes = testBlock.GetLocations();
int ipcport = nodes[0].GetIpcPort();
return(cluster.GetDataNode(ipcport));
}
/// <summary>
/// Reformatted DataNodes will replace the original UUID in the
/// <see cref="DatanodeManager#datanodeMap"/>
/// . This tests if block
/// invalidation work on the original DataNode can be skipped.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestDatanodeReformat()
{
namesystem.WriteLock();
try
{
// Change the datanode UUID to emulate a reformat
string poolId = cluster.GetNamesystem().GetBlockPoolId();
DatanodeRegistration dnr = cluster.GetDataNode(nodes[0].GetIpcPort()).GetDNRegistrationForBP
(poolId);
dnr = new DatanodeRegistration(UUID.RandomUUID().ToString(), dnr);
cluster.StopDataNode(nodes[0].GetXferAddr());
Block block = new Block(0, 0, GenerationStamp.LastReservedStamp);
bm.AddToInvalidates(block, nodes[0]);
bm.GetDatanodeManager().RegisterDatanode(dnr);
// Since UUID has changed, the invalidation work should be skipped
NUnit.Framework.Assert.AreEqual(0, bm.ComputeInvalidateWork(1));
NUnit.Framework.Assert.AreEqual(0, bm.GetPendingDeletionBlocksCount());
}
finally
{
namesystem.WriteUnlock();
}
}
public virtual void TestBlockSynchronization()
{
int OrgFileSize = 3000;
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(5).Build();
cluster.WaitActive();
//create a file
DistributedFileSystem dfs = cluster.GetFileSystem();
string filestr = "/foo";
Path filepath = new Path(filestr);
DFSTestUtil.CreateFile(dfs, filepath, OrgFileSize, ReplicationNum, 0L);
NUnit.Framework.Assert.IsTrue(dfs.Exists(filepath));
DFSTestUtil.WaitReplication(dfs, filepath, ReplicationNum);
//get block info for the last block
LocatedBlock locatedblock = TestInterDatanodeProtocol.GetLastLocatedBlock(dfs.dfs
.GetNamenode(), filestr);
DatanodeInfo[] datanodeinfos = locatedblock.GetLocations();
NUnit.Framework.Assert.AreEqual(ReplicationNum, datanodeinfos.Length);
//connect to data nodes
DataNode[] datanodes = new DataNode[ReplicationNum];
for (int i = 0; i < ReplicationNum; i++)
{
datanodes[i] = cluster.GetDataNode(datanodeinfos[i].GetIpcPort());
NUnit.Framework.Assert.IsTrue(datanodes[i] != null);
}
//verify Block Info
ExtendedBlock lastblock = locatedblock.GetBlock();
DataNode.Log.Info("newblocks=" + lastblock);
for (int i_1 = 0; i_1 < ReplicationNum; i_1++)
{
CheckMetaInfo(lastblock, datanodes[i_1]);
}
DataNode.Log.Info("dfs.dfs.clientName=" + dfs.dfs.clientName);
cluster.GetNameNodeRpc().Append(filestr, dfs.dfs.clientName, new EnumSetWritable <
CreateFlag>(EnumSet.Of(CreateFlag.Append)));
// expire lease to trigger block recovery.
WaitLeaseRecovery(cluster);
Block[] updatedmetainfo = new Block[ReplicationNum];
long oldSize = lastblock.GetNumBytes();
lastblock = TestInterDatanodeProtocol.GetLastLocatedBlock(dfs.dfs.GetNamenode(),
filestr).GetBlock();
long currentGS = lastblock.GetGenerationStamp();
for (int i_2 = 0; i_2 < ReplicationNum; i_2++)
{
updatedmetainfo[i_2] = DataNodeTestUtils.GetFSDataset(datanodes[i_2]).GetStoredBlock
(lastblock.GetBlockPoolId(), lastblock.GetBlockId());
NUnit.Framework.Assert.AreEqual(lastblock.GetBlockId(), updatedmetainfo[i_2].GetBlockId
());
NUnit.Framework.Assert.AreEqual(oldSize, updatedmetainfo[i_2].GetNumBytes());
NUnit.Framework.Assert.AreEqual(currentGS, updatedmetainfo[i_2].GetGenerationStamp
());
}
// verify that lease recovery does not occur when namenode is in safemode
System.Console.Out.WriteLine("Testing that lease recovery cannot happen during safemode."
);
filestr = "/foo.safemode";
filepath = new Path(filestr);
dfs.Create(filepath, (short)1);
cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter,
false);
NUnit.Framework.Assert.IsTrue(dfs.dfs.Exists(filestr));
DFSTestUtil.WaitReplication(dfs, filepath, (short)1);
WaitLeaseRecovery(cluster);
// verify that we still cannot recover the lease
LeaseManager lm = NameNodeAdapter.GetLeaseManager(cluster.GetNamesystem());
NUnit.Framework.Assert.IsTrue("Found " + lm.CountLease() + " lease, expected 1",
lm.CountLease() == 1);
cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave,
false);
}
/// <summary>Test race between delete operation and commitBlockSynchronization method.
/// </summary>
/// <remarks>
/// Test race between delete operation and commitBlockSynchronization method.
/// See HDFS-6825.
/// </remarks>
/// <param name="hasSnapshot"/>
/// <exception cref="System.Exception"/>
private void TestDeleteAndCommitBlockSynchronizationRace(bool hasSnapshot)
{
Log.Info("Start testing, hasSnapshot: " + hasSnapshot);
AList <AbstractMap.SimpleImmutableEntry <string, bool> > testList = new AList <AbstractMap.SimpleImmutableEntry
<string, bool> >();
testList.AddItem(new AbstractMap.SimpleImmutableEntry <string, bool>("/test-file",
false));
testList.AddItem(new AbstractMap.SimpleImmutableEntry <string, bool>("/test-file1"
, true));
testList.AddItem(new AbstractMap.SimpleImmutableEntry <string, bool>("/testdir/testdir1/test-file"
, false));
testList.AddItem(new AbstractMap.SimpleImmutableEntry <string, bool>("/testdir/testdir1/test-file1"
, true));
Path rootPath = new Path("/");
Configuration conf = new Configuration();
// Disable permissions so that another user can recover the lease.
conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, false);
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
FSDataOutputStream stm = null;
IDictionary <DataNode, DatanodeProtocolClientSideTranslatorPB> dnMap = new Dictionary
<DataNode, DatanodeProtocolClientSideTranslatorPB>();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
int stId = 0;
foreach (AbstractMap.SimpleImmutableEntry <string, bool> stest in testList)
{
string testPath = stest.Key;
bool mkSameDir = stest.Value;
Log.Info("test on " + testPath + " mkSameDir: " + mkSameDir + " snapshot: " + hasSnapshot
);
Path fPath = new Path(testPath);
//find grandest non-root parent
Path grandestNonRootParent = fPath;
while (!grandestNonRootParent.GetParent().Equals(rootPath))
{
grandestNonRootParent = grandestNonRootParent.GetParent();
}
stm = fs.Create(fPath);
Log.Info("test on " + testPath + " created " + fPath);
// write a half block
AppendTestUtil.Write(stm, 0, BlockSize / 2);
stm.Hflush();
if (hasSnapshot)
{
SnapshotTestHelper.CreateSnapshot(fs, rootPath, "st" + stId.ToString());
++stId;
}
// Look into the block manager on the active node for the block
// under construction.
NameNode nn = cluster.GetNameNode();
ExtendedBlock blk = DFSTestUtil.GetFirstBlock(fs, fPath);
DatanodeDescriptor expectedPrimary = DFSTestUtil.GetExpectedPrimaryNode(nn, blk);
Log.Info("Expecting block recovery to be triggered on DN " + expectedPrimary);
// Find the corresponding DN daemon, and spy on its connection to the
// active.
DataNode primaryDN = cluster.GetDataNode(expectedPrimary.GetIpcPort());
DatanodeProtocolClientSideTranslatorPB nnSpy = dnMap[primaryDN];
if (nnSpy == null)
{
nnSpy = DataNodeTestUtils.SpyOnBposToNN(primaryDN, nn);
dnMap[primaryDN] = nnSpy;
}
// Delay the commitBlockSynchronization call
GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log);
Org.Mockito.Mockito.DoAnswer(delayer).When(nnSpy).CommitBlockSynchronization(Org.Mockito.Mockito
.Eq(blk), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito
.Eq(true), Org.Mockito.Mockito.Eq(false), (DatanodeID[])Org.Mockito.Mockito.AnyObject
(), (string[])Org.Mockito.Mockito.AnyObject());
// new genstamp
// new length
// close file
// delete block
// new targets
// new target storages
fs.RecoverLease(fPath);
Log.Info("Waiting for commitBlockSynchronization call from primary");
delayer.WaitForCall();
Log.Info("Deleting recursively " + grandestNonRootParent);
fs.Delete(grandestNonRootParent, true);
if (mkSameDir && !grandestNonRootParent.ToString().Equals(testPath))
{
Log.Info("Recreate dir " + grandestNonRootParent + " testpath: " + testPath);
fs.Mkdirs(grandestNonRootParent);
}
delayer.Proceed();
Log.Info("Now wait for result");
delayer.WaitForResult();
Exception t = delayer.GetThrown();
if (t != null)
{
Log.Info("Result exception (snapshot: " + hasSnapshot + "): " + t);
}
}
// end of loop each fPath
Log.Info("Now check we can restart");
cluster.RestartNameNodes();
Log.Info("Restart finished");
}
finally
{
if (stm != null)
{
IOUtils.CloseStream(stm);
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}