public virtual void TestDataLocality() { Configuration conf = WebHdfsTestUtil.CreateConf(); string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1, Rack2, Rack2 }; int nDataNodes = racks.Length; Log.Info("nDataNodes=" + nDataNodes + ", racks=" + Arrays.AsList(racks)); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(nDataNodes ).Racks(racks).Build(); try { cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); NameNode namenode = cluster.GetNameNode(); DatanodeManager dm = namenode.GetNamesystem().GetBlockManager().GetDatanodeManager (); Log.Info("dm=" + dm); long blocksize = DFSConfigKeys.DfsBlockSizeDefault; string f = "/foo"; { //test CREATE for (int i = 0; i < nDataNodes; i++) { //set client address to a particular datanode DataNode dn = cluster.GetDataNodes()[i]; string ipAddr = dm.GetDatanode(dn.GetDatanodeId()).GetIpAddr(); //The chosen datanode must be the same as the client address DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, PutOpParam.OP .Create, -1L, blocksize, null); NUnit.Framework.Assert.AreEqual(ipAddr, chosen.GetIpAddr()); } } //create a file with one replica. Path p = new Path(f); FSDataOutputStream @out = dfs.Create(p, (short)1); @out.Write(1); @out.Close(); //get replica location. LocatedBlocks locatedblocks = NameNodeAdapter.GetBlockLocations(namenode, f, 0, 1 ); IList <LocatedBlock> lb = locatedblocks.GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(1, lb.Count); DatanodeInfo[] locations = lb[0].GetLocations(); NUnit.Framework.Assert.AreEqual(1, locations.Length); DatanodeInfo expected = locations[0]; { //For GETFILECHECKSUM, OPEN and APPEND, //the chosen datanode must be the same as the replica location. //test GETFILECHECKSUM DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP .Getfilechecksum, -1L, blocksize, null); NUnit.Framework.Assert.AreEqual(expected, chosen); } { //test OPEN DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP .Open, 0, blocksize, null); NUnit.Framework.Assert.AreEqual(expected, chosen); } { //test APPEND DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, PostOpParam.OP .Append, -1L, blocksize, null); NUnit.Framework.Assert.AreEqual(expected, chosen); } } finally { cluster.Shutdown(); } }
public virtual void TestExcludeDataNodes() { Configuration conf = WebHdfsTestUtil.CreateConf(); string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1, Rack2, Rack2 }; string[] hosts = new string[] { "DataNode1", "DataNode2", "DataNode3", "DataNode4" , "DataNode5", "DataNode6" }; int nDataNodes = hosts.Length; Log.Info("nDataNodes=" + nDataNodes + ", racks=" + Arrays.AsList(racks) + ", hosts=" + Arrays.AsList(hosts)); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Hosts(hosts).NumDataNodes (nDataNodes).Racks(racks).Build(); try { cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); NameNode namenode = cluster.GetNameNode(); DatanodeManager dm = namenode.GetNamesystem().GetBlockManager().GetDatanodeManager (); Log.Info("dm=" + dm); long blocksize = DFSConfigKeys.DfsBlockSizeDefault; string f = "/foo"; //create a file with three replica. Path p = new Path(f); FSDataOutputStream @out = dfs.Create(p, (short)3); @out.Write(1); @out.Close(); //get replica location. LocatedBlocks locatedblocks = NameNodeAdapter.GetBlockLocations(namenode, f, 0, 1 ); IList <LocatedBlock> lb = locatedblocks.GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(1, lb.Count); DatanodeInfo[] locations = lb[0].GetLocations(); NUnit.Framework.Assert.AreEqual(3, locations.Length); //For GETFILECHECKSUM, OPEN and APPEND, //the chosen datanode must be different with exclude nodes. StringBuilder sb = new StringBuilder(); for (int i = 0; i < 2; i++) { sb.Append(locations[i].GetXferAddr()); { // test GETFILECHECKSUM DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP .Getfilechecksum, -1L, blocksize, sb.ToString()); for (int j = 0; j <= i; j++) { Assert.AssertNotEquals(locations[j].GetHostName(), chosen.GetHostName()); } } { // test OPEN DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP .Open, 0, blocksize, sb.ToString()); for (int j = 0; j <= i; j++) { Assert.AssertNotEquals(locations[j].GetHostName(), chosen.GetHostName()); } } { // test APPEND DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, PostOpParam.OP .Append, -1L, blocksize, sb.ToString()); for (int j = 0; j <= i; j++) { Assert.AssertNotEquals(locations[j].GetHostName(), chosen.GetHostName()); } } sb.Append(","); } } finally { cluster.Shutdown(); } }