/* check if there are at least two nodes are on the same rack */ /// <exception cref="System.IO.IOException"/> private void CheckFile(FileSystem fileSys, Path name, int repl) { Configuration conf = fileSys.GetConf(); ClientProtocol namenode = NameNodeProxies.CreateProxy <ClientProtocol>(conf, fileSys .GetUri()).GetProxy(); WaitForBlockReplication(name.ToString(), namenode, Math.Min(numDatanodes, repl), -1); LocatedBlocks locations = namenode.GetBlockLocations(name.ToString(), 0, long.MaxValue ); FileStatus stat = fileSys.GetFileStatus(name); BlockLocation[] blockLocations = fileSys.GetFileBlockLocations(stat, 0L, long.MaxValue ); // verify that rack locations match NUnit.Framework.Assert.IsTrue(blockLocations.Length == locations.LocatedBlockCount ()); for (int i = 0; i < blockLocations.Length; i++) { LocatedBlock blk = locations.Get(i); DatanodeInfo[] datanodes = blk.GetLocations(); string[] topologyPaths = blockLocations[i].GetTopologyPaths(); NUnit.Framework.Assert.IsTrue(topologyPaths.Length == datanodes.Length); for (int j = 0; j < topologyPaths.Length; j++) { bool found = false; for (int k = 0; k < racks.Length; k++) { if (topologyPaths[j].StartsWith(racks[k])) { found = true; break; } } NUnit.Framework.Assert.IsTrue(found); } } bool isOnSameRack = true; bool isNotOnSameRack = true; foreach (LocatedBlock blk_1 in locations.GetLocatedBlocks()) { DatanodeInfo[] datanodes = blk_1.GetLocations(); if (datanodes.Length <= 1) { break; } if (datanodes.Length == 2) { isNotOnSameRack = !(datanodes[0].GetNetworkLocation().Equals(datanodes[1].GetNetworkLocation ())); break; } isOnSameRack = false; isNotOnSameRack = false; for (int i_1 = 0; i_1 < datanodes.Length - 1; i_1++) { Log.Info("datanode " + i_1 + ": " + datanodes[i_1]); bool onRack = false; for (int j = i_1 + 1; j < datanodes.Length; j++) { if (datanodes[i_1].GetNetworkLocation().Equals(datanodes[j].GetNetworkLocation())) { onRack = true; } } if (onRack) { isOnSameRack = true; } if (!onRack) { isNotOnSameRack = true; } if (isOnSameRack && isNotOnSameRack) { break; } } if (!isOnSameRack || !isNotOnSameRack) { break; } } NUnit.Framework.Assert.IsTrue(isOnSameRack); NUnit.Framework.Assert.IsTrue(isNotOnSameRack); }
public virtual void TestGetBlocks() { Configuration Conf = new HdfsConfiguration(); short ReplicationFactor = (short)2; int DefaultBlockSize = 1024; Random r = new Random(); Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize); MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(ReplicationFactor ).Build(); try { cluster.WaitActive(); // create a file with two blocks FileSystem fs = cluster.GetFileSystem(); FSDataOutputStream @out = fs.Create(new Path("/tmp.txt"), ReplicationFactor); byte[] data = new byte[1024]; long fileLen = 2 * DefaultBlockSize; long bytesToWrite = fileLen; while (bytesToWrite > 0) { r.NextBytes(data); int bytesToWriteNext = (1024 < bytesToWrite) ? 1024 : (int)bytesToWrite; @out.Write(data, 0, bytesToWriteNext); bytesToWrite -= bytesToWriteNext; } @out.Close(); // get blocks & data nodes IList <LocatedBlock> locatedBlocks; DatanodeInfo[] dataNodes = null; bool notWritten; do { DFSClient dfsclient = new DFSClient(NameNode.GetAddress(Conf), Conf); locatedBlocks = dfsclient.GetNamenode().GetBlockLocations("/tmp.txt", 0, fileLen) .GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(2, locatedBlocks.Count); notWritten = false; for (int i = 0; i < 2; i++) { dataNodes = locatedBlocks[i].GetLocations(); if (dataNodes.Length != ReplicationFactor) { notWritten = true; try { Sharpen.Thread.Sleep(10); } catch (Exception) { } break; } } }while (notWritten); // get RPC client to namenode IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort()); NamenodeProtocol namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(Conf, NameNode .GetUri(addr)).GetProxy(); // get blocks of size fileLen from dataNodes[0] BlocksWithLocations.BlockWithLocations[] locs; locs = namenode.GetBlocks(dataNodes[0], fileLen).GetBlocks(); NUnit.Framework.Assert.AreEqual(locs.Length, 2); NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2); NUnit.Framework.Assert.AreEqual(locs[1].GetStorageIDs().Length, 2); // get blocks of size BlockSize from dataNodes[0] locs = namenode.GetBlocks(dataNodes[0], DefaultBlockSize).GetBlocks(); NUnit.Framework.Assert.AreEqual(locs.Length, 1); NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2); // get blocks of size 1 from dataNodes[0] locs = namenode.GetBlocks(dataNodes[0], 1).GetBlocks(); NUnit.Framework.Assert.AreEqual(locs.Length, 1); NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2); // get blocks of size 0 from dataNodes[0] GetBlocksWithException(namenode, dataNodes[0], 0); // get blocks of size -1 from dataNodes[0] GetBlocksWithException(namenode, dataNodes[0], -1); // get blocks of size BlockSize from a non-existent datanode DatanodeInfo info = DFSTestUtil.GetDatanodeInfo("1.2.3.4"); GetBlocksWithException(namenode, info, 2); } finally { cluster.Shutdown(); } }