public virtual void TestAbandonBlock()
{
string src = FileNamePrefix + "foo";
// Start writing a file but do not close it
FSDataOutputStream fout = fs.Create(new Path(src), true, 4096, (short)1, 512L);
for (int i = 0; i < 1024; i++)
{
fout.Write(123);
}
fout.Hflush();
long fileId = ((DFSOutputStream)fout.GetWrappedStream()).GetFileId();
// Now abandon the last block
DFSClient dfsclient = DFSClientAdapter.GetDFSClient(fs);
LocatedBlocks blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue
);
int orginalNumBlocks = blocks.LocatedBlockCount();
LocatedBlock b = blocks.GetLastLocatedBlock();
dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName
);
// call abandonBlock again to make sure the operation is idempotent
dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName
);
// And close the file
fout.Close();
// Close cluster and check the block has been abandoned after restart
cluster.RestartNameNode();
blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue);
NUnit.Framework.Assert.AreEqual("Blocks " + b + " has not been abandoned.", orginalNumBlocks
, blocks.LocatedBlockCount() + 1);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.TimeoutException"/>
private void ChangeBlockLen(MiniDFSCluster cluster, int lenDelta)
{
Path fileName = new Path("/file1");
short ReplicationFactor = (short)1;
FileSystem fs = cluster.GetFileSystem();
int fileLen = fs.GetConf().GetInt(DFSConfigKeys.DfsBytesPerChecksumKey, 512);
DFSTestUtil.CreateFile(fs, fileName, fileLen, ReplicationFactor, 0);
DFSTestUtil.WaitReplication(fs, fileName, ReplicationFactor);
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName);
// Change the length of a replica
for (int i = 0; i < cluster.GetDataNodes().Count; i++)
{
if (DFSTestUtil.ChangeReplicaLength(cluster, block, i, lenDelta))
{
break;
}
}
// increase the file's replication factor
fs.SetReplication(fileName, (short)(ReplicationFactor + 1));
// block replication triggers corrupt block detection
DFSClient dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort
()), fs.GetConf());
LocatedBlocks blocks = dfsClient.GetNamenode().GetBlockLocations(fileName.ToString
(), 0, fileLen);
if (lenDelta < 0)
{
// replica truncated
while (!blocks.Get(0).IsCorrupt() || ReplicationFactor != blocks.Get(0).GetLocations
().Length)
{
Sharpen.Thread.Sleep(100);
blocks = dfsClient.GetNamenode().GetBlockLocations(fileName.ToString(), 0, fileLen
);
}
}
else
{
// no corruption detected; block replicated
while (ReplicationFactor + 1 != blocks.Get(0).GetLocations().Length)
{
Sharpen.Thread.Sleep(100);
blocks = dfsClient.GetNamenode().GetBlockLocations(fileName.ToString(), 0, fileLen
);
}
}
fs.Delete(fileName, true);
}
/// <exception cref="System.Exception"/>
private void TestBadBlockReportOnTransfer(bool corruptBlockByDeletingBlockFile)
{
Configuration conf = new HdfsConfiguration();
FileSystem fs = null;
DFSClient dfsClient = null;
LocatedBlocks blocks = null;
int replicaCount = 0;
short replFactor = 1;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()),
conf);
// Create file with replication factor of 1
Path file1 = new Path("/tmp/testBadBlockReportOnTransfer/file1");
DFSTestUtil.CreateFile(fs, file1, 1024, replFactor, 0);
DFSTestUtil.WaitReplication(fs, file1, replFactor);
// Corrupt the block belonging to the created file
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, file1);
int blockFilesCorrupted = corruptBlockByDeletingBlockFile ? cluster.CorruptBlockOnDataNodesByDeletingBlockFile
(block) : cluster.CorruptBlockOnDataNodes(block);
NUnit.Framework.Assert.AreEqual("Corrupted too few blocks", replFactor, blockFilesCorrupted
);
// Increase replication factor, this should invoke transfer request
// Receiving datanode fails on checksum and reports it to namenode
replFactor = 2;
fs.SetReplication(file1, replFactor);
// Now get block details and check if the block is corrupt
blocks = dfsClient.GetNamenode().GetBlockLocations(file1.ToString(), 0, long.MaxValue
);
while (blocks.Get(0).IsCorrupt() != true)
{
try
{
Log.Info("Waiting until block is marked as corrupt...");
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
blocks = dfsClient.GetNamenode().GetBlockLocations(file1.ToString(), 0, long.MaxValue
);
}
replicaCount = blocks.Get(0).GetLocations().Length;
NUnit.Framework.Assert.IsTrue(replicaCount == 1);
cluster.Shutdown();
}
/// <summary>Get the list of Blocks for a file.</summary>
/// <exception cref="System.IO.IOException"/>
public virtual IList <LocatedBlock> GetFileBlocks(Path filepath, int sizeKB)
{
// Return the blocks we just wrote
DFSClient dfsclient = GetDFSClient();
return(dfsclient.GetNamenode().GetBlockLocations(filepath.ToString(), 0, sizeKB *
1024).GetLocatedBlocks());
}
/// <summary>Get the internet address of the currently-active NN.</summary>
/// <remarks>
/// Get the internet address of the currently-active NN. This should rarely be
/// used, since callers of this method who connect directly to the NN using the
/// resulting InetSocketAddress will not be able to connect to the active NN if
/// a failover were to occur after this method has been called.
/// </remarks>
/// <param name="fs">the file system to get the active address of.</param>
/// <returns>the internet address of the currently-active NN.</returns>
/// <exception cref="System.IO.IOException">if an error occurs while resolving the active NN.
/// </exception>
public static IPEndPoint GetAddressOfActive(FileSystem fs)
{
if (!(fs is DistributedFileSystem))
{
throw new ArgumentException("FileSystem " + fs + " is not a DFS.");
}
// force client address resolution.
fs.Exists(new Path("/"));
DistributedFileSystem dfs = (DistributedFileSystem)fs;
DFSClient dfsClient = dfs.GetClient();
return(RPC.GetServerAddress(dfsClient.GetNamenode()));
}
public virtual void TestPendingReplicationRetry()
{
MiniDFSCluster cluster = null;
int numDataNodes = 4;
string testFile = "/replication-test-file";
Path testPath = new Path(testFile);
byte[] buffer = new byte[1024];
for (int i = 0; i < buffer.Length; i++)
{
buffer[i] = (byte)('1');
}
try
{
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsReplicationKey, Sharpen.Extensions.ToString(numDataNodes
));
//first time format
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
DFSClient dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort
()), conf);
OutputStream @out = cluster.GetFileSystem().Create(testPath);
@out.Write(buffer);
@out.Close();
WaitForBlockReplication(testFile, dfsClient.GetNamenode(), numDataNodes, -1);
// get first block of the file.
ExtendedBlock block = dfsClient.GetNamenode().GetBlockLocations(testFile, 0, long.MaxValue
).Get(0).GetBlock();
cluster.Shutdown();
for (int i_1 = 0; i_1 < 25; i_1++)
{
buffer[i_1] = (byte)('0');
}
int fileCount = 0;
// Choose 3 copies of block file - delete 1 and corrupt the remaining 2
for (int dnIndex = 0; dnIndex < 3; dnIndex++)
{
FilePath blockFile = cluster.GetBlockFile(dnIndex, block);
Log.Info("Checking for file " + blockFile);
if (blockFile != null && blockFile.Exists())
{
if (fileCount == 0)
{
Log.Info("Deleting file " + blockFile);
NUnit.Framework.Assert.IsTrue(blockFile.Delete());
}
else
{
// corrupt it.
Log.Info("Corrupting file " + blockFile);
long len = blockFile.Length();
NUnit.Framework.Assert.IsTrue(len > 50);
RandomAccessFile blockOut = new RandomAccessFile(blockFile, "rw");
try
{
blockOut.Seek(len / 3);
blockOut.Write(buffer, 0, 25);
}
finally
{
blockOut.Close();
}
}
fileCount++;
}
}
NUnit.Framework.Assert.AreEqual(3, fileCount);
/* Start the MiniDFSCluster with more datanodes since once a writeBlock
* to a datanode node fails, same block can not be written to it
* immediately. In our case some replication attempts will fail.
*/
Log.Info("Restarting minicluster after deleting a replica and corrupting 2 crcs");
conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsReplicationKey, Sharpen.Extensions.ToString(numDataNodes
));
conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString
(2));
conf.Set("dfs.datanode.block.write.timeout.sec", Sharpen.Extensions.ToString(5));
conf.Set(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, "0.75f");
// only 3 copies exist
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes * 2).Format(
false).Build();
cluster.WaitActive();
dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()),
conf);
WaitForBlockReplication(testFile, dfsClient.GetNamenode(), numDataNodes, -1);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestInjection()
{
MiniDFSCluster cluster = null;
string testFile = "/replication-test-file";
Path testPath = new Path(testFile);
byte[] buffer = new byte[1024];
for (int i = 0; i < buffer.Length; i++)
{
buffer[i] = (byte)('1');
}
try
{
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsReplicationKey, Sharpen.Extensions.ToString(numDataNodes
));
conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, checksumSize);
SimulatedFSDataset.SetFactory(conf);
//first time format
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
string bpid = cluster.GetNamesystem().GetBlockPoolId();
DFSClient dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort
()), conf);
WriteFile(cluster.GetFileSystem(), testPath, numDataNodes);
WaitForBlockReplication(testFile, dfsClient.GetNamenode(), numDataNodes, 20);
IList <IDictionary <DatanodeStorage, BlockListAsLongs> > blocksList = cluster.GetAllBlockReports
(bpid);
cluster.Shutdown();
cluster = null;
/* Start the MiniDFSCluster with more datanodes since once a writeBlock
* to a datanode node fails, same block can not be written to it
* immediately. In our case some replication attempts will fail.
*/
Log.Info("Restarting minicluster");
conf = new HdfsConfiguration();
SimulatedFSDataset.SetFactory(conf);
conf.Set(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, "0.0f");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes * 2).Format(
false).Build();
cluster.WaitActive();
ICollection <Block> uniqueBlocks = new HashSet <Block>();
foreach (IDictionary <DatanodeStorage, BlockListAsLongs> map in blocksList)
{
foreach (BlockListAsLongs blockList in map.Values)
{
foreach (Block b in blockList)
{
uniqueBlocks.AddItem(new Block(b));
}
}
}
// Insert all the blocks in the first data node
Log.Info("Inserting " + uniqueBlocks.Count + " blocks");
cluster.InjectBlocks(0, uniqueBlocks, null);
dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()),
conf);
WaitForBlockReplication(testFile, dfsClient.GetNamenode(), numDataNodes, -1);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestBlockReplacement()
{
Configuration Conf = new HdfsConfiguration();
string[] InitialRacks = new string[] { "/RACK0", "/RACK1", "/RACK2" };
string[] NewRacks = new string[] { "/RACK2" };
short ReplicationFactor = (short)3;
int DefaultBlockSize = 1024;
Random r = new Random();
Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
Conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, DefaultBlockSize / 2);
Conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 500);
cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(ReplicationFactor).Racks(
InitialRacks).Build();
try
{
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
Path fileName = new Path("/tmp.txt");
// create a file with one block
DFSTestUtil.CreateFile(fs, fileName, DefaultBlockSize, ReplicationFactor, r.NextLong
());
DFSTestUtil.WaitReplication(fs, fileName, ReplicationFactor);
// get all datanodes
IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort());
DFSClient client = new DFSClient(addr, Conf);
IList <LocatedBlock> locatedBlocks = client.GetNamenode().GetBlockLocations("/tmp.txt"
, 0, DefaultBlockSize).GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, locatedBlocks.Count);
LocatedBlock block = locatedBlocks[0];
DatanodeInfo[] oldNodes = block.GetLocations();
NUnit.Framework.Assert.AreEqual(oldNodes.Length, 3);
ExtendedBlock b = block.GetBlock();
// add a fourth datanode to the cluster
cluster.StartDataNodes(Conf, 1, true, null, NewRacks);
cluster.WaitActive();
DatanodeInfo[] datanodes = client.DatanodeReport(HdfsConstants.DatanodeReportType
.All);
// find out the new node
DatanodeInfo newNode = null;
foreach (DatanodeInfo node in datanodes)
{
bool isNewNode = true;
foreach (DatanodeInfo oldNode in oldNodes)
{
if (node.Equals(oldNode))
{
isNewNode = false;
break;
}
}
if (isNewNode)
{
newNode = node;
break;
}
}
NUnit.Framework.Assert.IsTrue(newNode != null);
DatanodeInfo source = null;
AList <DatanodeInfo> proxies = new AList <DatanodeInfo>(2);
foreach (DatanodeInfo node_1 in datanodes)
{
if (node_1 != newNode)
{
if (node_1.GetNetworkLocation().Equals(newNode.GetNetworkLocation()))
{
source = node_1;
}
else
{
proxies.AddItem(node_1);
}
}
}
//current state: the newNode is on RACK2, and "source" is the other dn on RACK2.
//the two datanodes on RACK0 and RACK1 are in "proxies".
//"source" and both "proxies" all contain the block, while newNode doesn't yet.
NUnit.Framework.Assert.IsTrue(source != null && proxies.Count == 2);
// start to replace the block
// case 1: proxySource does not contain the block
Log.Info("Testcase 1: Proxy " + newNode + " does not contain the block " + b);
NUnit.Framework.Assert.IsFalse(ReplaceBlock(b, source, newNode, proxies[0]));
// case 2: destination already contains the block
Log.Info("Testcase 2: Destination " + proxies[1] + " contains the block " + b);
NUnit.Framework.Assert.IsFalse(ReplaceBlock(b, source, proxies[0], proxies[1]));
// case 3: correct case
Log.Info("Testcase 3: Source=" + source + " Proxy=" + proxies[0] + " Destination="
+ newNode);
NUnit.Framework.Assert.IsTrue(ReplaceBlock(b, source, proxies[0], newNode));
// after cluster has time to resolve the over-replication,
// block locations should contain two proxies and newNode
// but not source
CheckBlocks(new DatanodeInfo[] { newNode, proxies[0], proxies[1] }, fileName.ToString
(), DefaultBlockSize, ReplicationFactor, client);
// case 4: proxies.get(0) is not a valid del hint
// expect either source or newNode replica to be deleted instead
Log.Info("Testcase 4: invalid del hint " + proxies[0]);
NUnit.Framework.Assert.IsTrue(ReplaceBlock(b, proxies[0], proxies[1], source));
// after cluster has time to resolve the over-replication,
// block locations should contain two proxies,
// and either source or newNode, but not both.
CheckBlocks(Sharpen.Collections.ToArray(proxies, new DatanodeInfo[proxies.Count])
, fileName.ToString(), DefaultBlockSize, ReplicationFactor, client);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestDeletedBlockWhenAddBlockIsInEdit()
{
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(1).Build();
DFSClient client = null;
try
{
cluster.WaitActive();
NUnit.Framework.Assert.AreEqual("Number of namenodes is not 2", 2, cluster.GetNumNameNodes
());
// Transitioning the namenode 0 to active.
cluster.TransitionToActive(0);
NUnit.Framework.Assert.IsTrue("Namenode 0 should be in active state", cluster.GetNameNode
(0).IsActiveState());
NUnit.Framework.Assert.IsTrue("Namenode 1 should be in standby state", cluster.GetNameNode
(1).IsStandbyState());
// Trigger heartbeat to mark DatanodeStorageInfo#heartbeatedSinceFailover
// to true.
DataNodeTestUtils.TriggerHeartbeat(cluster.GetDataNodes()[0]);
FileSystem fs = cluster.GetFileSystem(0);
// Trigger blockReport to mark DatanodeStorageInfo#blockContentsStale
// to false.
cluster.GetDataNodes()[0].TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental
(false).Build());
Path fileName = new Path("/tmp.txt");
// create a file with one block
DFSTestUtil.CreateFile(fs, fileName, 10L, (short)1, 1234L);
DFSTestUtil.WaitReplication(fs, fileName, (short)1);
client = new DFSClient(cluster.GetFileSystem(0).GetUri(), conf);
IList <LocatedBlock> locatedBlocks = client.GetNamenode().GetBlockLocations("/tmp.txt"
, 0, 10L).GetLocatedBlocks();
NUnit.Framework.Assert.IsTrue(locatedBlocks.Count == 1);
NUnit.Framework.Assert.IsTrue(locatedBlocks[0].GetLocations().Length == 1);
// add a second datanode to the cluster
cluster.StartDataNodes(conf, 1, true, null, null, null, null);
NUnit.Framework.Assert.AreEqual("Number of datanodes should be 2", 2, cluster.GetDataNodes
().Count);
DataNode dn0 = cluster.GetDataNodes()[0];
DataNode dn1 = cluster.GetDataNodes()[1];
string activeNNBPId = cluster.GetNamesystem(0).GetBlockPoolId();
DatanodeDescriptor sourceDnDesc = NameNodeAdapter.GetDatanode(cluster.GetNamesystem
(0), dn0.GetDNRegistrationForBP(activeNNBPId));
DatanodeDescriptor destDnDesc = NameNodeAdapter.GetDatanode(cluster.GetNamesystem
(0), dn1.GetDNRegistrationForBP(activeNNBPId));
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName);
Log.Info("replaceBlock: " + ReplaceBlock(block, (DatanodeInfo)sourceDnDesc, (DatanodeInfo
)sourceDnDesc, (DatanodeInfo)destDnDesc));
// Waiting for the FsDatasetAsyncDsikService to delete the block
Sharpen.Thread.Sleep(3000);
// Triggering the incremental block report to report the deleted block to
// namnemode
cluster.GetDataNodes()[0].TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental
(true).Build());
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
NUnit.Framework.Assert.IsTrue("Namenode 1 should be in active state", cluster.GetNameNode
(1).IsActiveState());
NUnit.Framework.Assert.IsTrue("Namenode 0 should be in standby state", cluster.GetNameNode
(0).IsStandbyState());
client.Close();
// Opening a new client for new active namenode
client = new DFSClient(cluster.GetFileSystem(1).GetUri(), conf);
IList <LocatedBlock> locatedBlocks1 = client.GetNamenode().GetBlockLocations("/tmp.txt"
, 0, 10L).GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, locatedBlocks1.Count);
NUnit.Framework.Assert.AreEqual("The block should be only on 1 datanode ", 1, locatedBlocks1
[0].GetLocations().Length);
}
finally
{
IOUtils.Cleanup(null, client);
cluster.Shutdown();
}
}
/* check if file's blocks have expected number of replicas,
* and exist at all of includeNodes
*/
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.TimeoutException"/>
private void CheckBlocks(DatanodeInfo[] includeNodes, string fileName, long fileLen
, short replFactor, DFSClient client)
{
bool notDone;
long Timeout = 20000L;
long starttime = Time.MonotonicNow();
long failtime = starttime + Timeout;
do
{
try
{
Sharpen.Thread.Sleep(100);
}
catch (Exception)
{
}
IList <LocatedBlock> blocks = client.GetNamenode().GetBlockLocations(fileName, 0,
fileLen).GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, blocks.Count);
DatanodeInfo[] nodes = blocks[0].GetLocations();
notDone = (nodes.Length != replFactor);
if (notDone)
{
Log.Info("Expected replication factor is " + replFactor + " but the real replication factor is "
+ nodes.Length);
}
else
{
IList <DatanodeInfo> nodeLocations = Arrays.AsList(nodes);
foreach (DatanodeInfo node in includeNodes)
{
if (!nodeLocations.Contains(node))
{
notDone = true;
Log.Info("Block is not located at " + node);
break;
}
}
}
if (Time.MonotonicNow() > failtime)
{
string expectedNodesList = string.Empty;
string currentNodesList = string.Empty;
foreach (DatanodeInfo dn in includeNodes)
{
expectedNodesList += dn + ", ";
}
foreach (DatanodeInfo dn_1 in nodes)
{
currentNodesList += dn_1 + ", ";
}
Log.Info("Expected replica nodes are: " + expectedNodesList);
Log.Info("Current actual replica nodes are: " + currentNodesList);
throw new TimeoutException("Did not achieve expected replication to expected nodes "
+ "after more than " + Timeout + " msec. See logs for details.");
}
}while (notDone);
Log.Info("Achieved expected replication values in " + (Time.Now() - starttime) +
" msec.");
}
public virtual void TestReadSelectNonStaleDatanode()
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForReadKey, true);
long staleInterval = 30 * 1000 * 60;
conf.SetLong(DFSConfigKeys.DfsNamenodeStaleDatanodeIntervalKey, staleInterval);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes
).Racks(racks).Build();
cluster.WaitActive();
IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort());
DFSClient client = new DFSClient(addr, conf);
IList <DatanodeDescriptor> nodeInfoList = cluster.GetNameNode().GetNamesystem().GetBlockManager
().GetDatanodeManager().GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Live);
NUnit.Framework.Assert.AreEqual("Unexpected number of datanodes", numDatanodes, nodeInfoList
.Count);
FileSystem fileSys = cluster.GetFileSystem();
FSDataOutputStream stm = null;
try
{
// do the writing but do not close the FSDataOutputStream
// in order to mimic the ongoing writing
Path fileName = new Path("/file1");
stm = fileSys.Create(fileName, true, fileSys.GetConf().GetInt(CommonConfigurationKeys
.IoFileBufferSizeKey, 4096), (short)3, blockSize);
stm.Write(new byte[(blockSize * 3) / 2]);
// We do not close the stream so that
// the writing seems to be still ongoing
stm.Hflush();
LocatedBlocks blocks = client.GetNamenode().GetBlockLocations(fileName.ToString()
, 0, blockSize);
DatanodeInfo[] nodes = blocks.Get(0).GetLocations();
NUnit.Framework.Assert.AreEqual(nodes.Length, 3);
DataNode staleNode = null;
DatanodeDescriptor staleNodeInfo = null;
// stop the heartbeat of the first node
staleNode = this.StopDataNodeHeartbeat(cluster, nodes[0].GetHostName());
NUnit.Framework.Assert.IsNotNull(staleNode);
// set the first node as stale
staleNodeInfo = cluster.GetNameNode().GetNamesystem().GetBlockManager().GetDatanodeManager
().GetDatanode(staleNode.GetDatanodeId());
DFSTestUtil.ResetLastUpdatesWithOffset(staleNodeInfo, -(staleInterval + 1));
LocatedBlocks blocksAfterStale = client.GetNamenode().GetBlockLocations(fileName.
ToString(), 0, blockSize);
DatanodeInfo[] nodesAfterStale = blocksAfterStale.Get(0).GetLocations();
NUnit.Framework.Assert.AreEqual(nodesAfterStale.Length, 3);
NUnit.Framework.Assert.AreEqual(nodesAfterStale[2].GetHostName(), nodes[0].GetHostName
());
// restart the staleNode's heartbeat
DataNodeTestUtils.SetHeartbeatsDisabledForTests(staleNode, false);
// reset the first node as non-stale, so as to avoid two stale nodes
DFSTestUtil.ResetLastUpdatesWithOffset(staleNodeInfo, 0);
LocatedBlock lastBlock = client.GetLocatedBlocks(fileName.ToString(), 0, long.MaxValue
).GetLastLocatedBlock();
nodes = lastBlock.GetLocations();
NUnit.Framework.Assert.AreEqual(nodes.Length, 3);
// stop the heartbeat of the first node for the last block
staleNode = this.StopDataNodeHeartbeat(cluster, nodes[0].GetHostName());
NUnit.Framework.Assert.IsNotNull(staleNode);
// set the node as stale
DatanodeDescriptor dnDesc = cluster.GetNameNode().GetNamesystem().GetBlockManager
().GetDatanodeManager().GetDatanode(staleNode.GetDatanodeId());
DFSTestUtil.ResetLastUpdatesWithOffset(dnDesc, -(staleInterval + 1));
LocatedBlock lastBlockAfterStale = client.GetLocatedBlocks(fileName.ToString(), 0
, long.MaxValue).GetLastLocatedBlock();
nodesAfterStale = lastBlockAfterStale.GetLocations();
NUnit.Framework.Assert.AreEqual(nodesAfterStale.Length, 3);
NUnit.Framework.Assert.AreEqual(nodesAfterStale[2].GetHostName(), nodes[0].GetHostName
());
}
finally
{
if (stm != null)
{
stm.Close();
}
client.Close();
cluster.Shutdown();
}
}
public virtual void TestGetBlocks()
{
Configuration Conf = new HdfsConfiguration();
short ReplicationFactor = (short)2;
int DefaultBlockSize = 1024;
Random r = new Random();
Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(ReplicationFactor
).Build();
try
{
cluster.WaitActive();
// create a file with two blocks
FileSystem fs = cluster.GetFileSystem();
FSDataOutputStream @out = fs.Create(new Path("/tmp.txt"), ReplicationFactor);
byte[] data = new byte[1024];
long fileLen = 2 * DefaultBlockSize;
long bytesToWrite = fileLen;
while (bytesToWrite > 0)
{
r.NextBytes(data);
int bytesToWriteNext = (1024 < bytesToWrite) ? 1024 : (int)bytesToWrite;
@out.Write(data, 0, bytesToWriteNext);
bytesToWrite -= bytesToWriteNext;
}
@out.Close();
// get blocks & data nodes
IList <LocatedBlock> locatedBlocks;
DatanodeInfo[] dataNodes = null;
bool notWritten;
do
{
DFSClient dfsclient = new DFSClient(NameNode.GetAddress(Conf), Conf);
locatedBlocks = dfsclient.GetNamenode().GetBlockLocations("/tmp.txt", 0, fileLen)
.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(2, locatedBlocks.Count);
notWritten = false;
for (int i = 0; i < 2; i++)
{
dataNodes = locatedBlocks[i].GetLocations();
if (dataNodes.Length != ReplicationFactor)
{
notWritten = true;
try
{
Sharpen.Thread.Sleep(10);
}
catch (Exception)
{
}
break;
}
}
}while (notWritten);
// get RPC client to namenode
IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort());
NamenodeProtocol namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(Conf, NameNode
.GetUri(addr)).GetProxy();
// get blocks of size fileLen from dataNodes[0]
BlocksWithLocations.BlockWithLocations[] locs;
locs = namenode.GetBlocks(dataNodes[0], fileLen).GetBlocks();
NUnit.Framework.Assert.AreEqual(locs.Length, 2);
NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
NUnit.Framework.Assert.AreEqual(locs[1].GetStorageIDs().Length, 2);
// get blocks of size BlockSize from dataNodes[0]
locs = namenode.GetBlocks(dataNodes[0], DefaultBlockSize).GetBlocks();
NUnit.Framework.Assert.AreEqual(locs.Length, 1);
NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
// get blocks of size 1 from dataNodes[0]
locs = namenode.GetBlocks(dataNodes[0], 1).GetBlocks();
NUnit.Framework.Assert.AreEqual(locs.Length, 1);
NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
// get blocks of size 0 from dataNodes[0]
GetBlocksWithException(namenode, dataNodes[0], 0);
// get blocks of size -1 from dataNodes[0]
GetBlocksWithException(namenode, dataNodes[0], -1);
// get blocks of size BlockSize from a non-existent datanode
DatanodeInfo info = DFSTestUtil.GetDatanodeInfo("1.2.3.4");
GetBlocksWithException(namenode, info, 2);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestRestartDfsWithAbandonedBlock()
{
Configuration conf = new HdfsConfiguration();
// Turn off persistent IPC, so that the DFSClient can survive NN restart
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0);
MiniDFSCluster cluster = null;
long len = 0;
FSDataOutputStream stream;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
FileSystem fs = cluster.GetFileSystem();
// Creating a file with 4096 blockSize to write multiple blocks
stream = fs.Create(FilePath, true, BlockSize, (short)1, BlockSize);
stream.Write(DataBeforeRestart);
stream.Hflush();
// Wait for all of the blocks to get through
while (len < BlockSize * (NumBlocks - 1))
{
FileStatus status = fs.GetFileStatus(FilePath);
len = status.GetLen();
Sharpen.Thread.Sleep(100);
}
// Abandon the last block
DFSClient dfsclient = DFSClientAdapter.GetDFSClient((DistributedFileSystem)fs);
HdfsFileStatus fileStatus = dfsclient.GetNamenode().GetFileInfo(FileName);
LocatedBlocks blocks = dfsclient.GetNamenode().GetBlockLocations(FileName, 0, BlockSize
* NumBlocks);
NUnit.Framework.Assert.AreEqual(NumBlocks, blocks.GetLocatedBlocks().Count);
LocatedBlock b = blocks.GetLastLocatedBlock();
dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileStatus.GetFileId(), FileName
, dfsclient.clientName);
// explicitly do NOT close the file.
cluster.RestartNameNode();
// Check that the file has no less bytes than before the restart
// This would mean that blocks were successfully persisted to the log
FileStatus status_1 = fs.GetFileStatus(FilePath);
NUnit.Framework.Assert.IsTrue("Length incorrect: " + status_1.GetLen(), status_1.
GetLen() == len - BlockSize);
// Verify the data showed up from before restart, sans abandoned block.
FSDataInputStream readStream = fs.Open(FilePath);
try
{
byte[] verifyBuf = new byte[DataBeforeRestart.Length - BlockSize];
IOUtils.ReadFully(readStream, verifyBuf, 0, verifyBuf.Length);
byte[] expectedBuf = new byte[DataBeforeRestart.Length - BlockSize];
System.Array.Copy(DataBeforeRestart, 0, expectedBuf, 0, expectedBuf.Length);
Assert.AssertArrayEquals(expectedBuf, verifyBuf);
}
finally
{
IOUtils.CloseStream(readStream);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestCopyOnWrite()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort());
DFSClient client = new DFSClient(addr, conf);
try
{
// create a new file, write to it and close it.
//
Path file1 = new Path("/filestatus.dat");
FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1);
WriteFile(stm);
stm.Close();
// Get a handle to the datanode
DataNode[] dn = cluster.ListDataNodes();
NUnit.Framework.Assert.IsTrue("There should be only one datanode but found " + dn
.Length, dn.Length == 1);
LocatedBlocks locations = client.GetNamenode().GetBlockLocations(file1.ToString()
, 0, long.MaxValue);
IList <LocatedBlock> blocks = locations.GetLocatedBlocks();
//
// Create hard links for a few of the blocks
//
for (int i = 0; i < blocks.Count; i = i + 2)
{
ExtendedBlock b = blocks[i].GetBlock();
FilePath f = DataNodeTestUtils.GetFile(dn[0], b.GetBlockPoolId(), b.GetLocalBlock
().GetBlockId());
FilePath link = new FilePath(f.ToString() + ".link");
System.Console.Out.WriteLine("Creating hardlink for File " + f + " to " + link);
HardLink.CreateHardLink(f, link);
}
//
// Detach all blocks. This should remove hardlinks (if any)
//
for (int i_1 = 0; i_1 < blocks.Count; i_1++)
{
ExtendedBlock b = blocks[i_1].GetBlock();
System.Console.Out.WriteLine("testCopyOnWrite detaching block " + b);
NUnit.Framework.Assert.IsTrue("Detaching block " + b + " should have returned true"
, DataNodeTestUtils.UnlinkBlock(dn[0], b, 1));
}
// Since the blocks were already detached earlier, these calls should
// return false
//
for (int i_2 = 0; i_2 < blocks.Count; i_2++)
{
ExtendedBlock b = blocks[i_2].GetBlock();
System.Console.Out.WriteLine("testCopyOnWrite detaching block " + b);
NUnit.Framework.Assert.IsTrue("Detaching block " + b + " should have returned false"
, !DataNodeTestUtils.UnlinkBlock(dn[0], b, 1));
}
}
finally
{
client.Close();
fs.Close();
cluster.Shutdown();
}
}