/// <summary>Create a redirection URL</summary> /// <exception cref="System.IO.IOException"/> private Uri CreateRedirectURL(string path, string encodedPath, HdfsFileStatus status , UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request, string dt) { string scheme = request.GetScheme(); LocatedBlocks blks = nnproxy.GetBlockLocations(status.GetFullPath(new Path(path)) .ToUri().GetPath(), 0, 1); Configuration conf = NameNodeHttpServer.GetConfFromContext(GetServletContext()); DatanodeID host = PickSrcDatanode(blks, status, conf); string hostname; if (host is DatanodeInfo) { hostname = host.GetHostName(); } else { hostname = host.GetIpAddr(); } int port = "https".Equals(scheme) ? host.GetInfoSecurePort() : host.GetInfoPort(); string dtParam = string.Empty; if (dt != null) { dtParam = JspHelper.GetDelegationTokenUrlParam(dt); } // Add namenode address to the url params NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext()); string addr = nn.GetNameNodeAddressHostPortString(); string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr); return(new Uri(scheme, hostname, port, "/streamFile" + encodedPath + '?' + "ugi=" + ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam)); }
/// <summary> /// Create a cluster with even distribution, and a new empty node is added to /// the cluster, then test rack locality for balancer policy. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestBalancerWithRackLocality() { Configuration conf = CreateConf(); long[] capacities = new long[] { Capacity, Capacity }; string[] racks = new string[] { Rack0, Rack1 }; string[] nodeGroups = new string[] { Nodegroup0, Nodegroup1 }; int numOfDatanodes = capacities.Length; NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities .Length).Racks(racks).SimulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder); try { cluster.WaitActive(); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem( 0).GetUri()).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity * 3 / 10; long length = totalUsedSpace / numOfDatanodes; TestBalancer.CreateFile(cluster, filePath, length, (short)numOfDatanodes, 0); LocatedBlocks lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length ); ICollection <ExtendedBlock> before = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0 ); long newCapacity = Capacity; string newRack = Rack1; string newNodeGroup = Nodegroup2; // start up an empty node with the same capacity and on the same rack cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[] { newCapacity }, new string[] { newNodeGroup }); totalCapacity += newCapacity; // run balancer and validate results RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity); lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length); ICollection <ExtendedBlock> after = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0); NUnit.Framework.Assert.AreEqual(before, after); } finally { cluster.Shutdown(); } }
// Waits for all of the blocks to have expected replication // Waits for all of the blocks to have expected replication /// <exception cref="System.IO.IOException"/> private void WaitForBlockReplication(string filename, ClientProtocol namenode, int expected, long maxWaitSec) { long start = Time.MonotonicNow(); //wait for all the blocks to be replicated; Log.Info("Checking for block replication for " + filename); LocatedBlocks blocks = namenode.GetBlockLocations(filename, 0, long.MaxValue); NUnit.Framework.Assert.AreEqual(numBlocks, blocks.LocatedBlockCount()); for (int i = 0; i < numBlocks; ++i) { Log.Info("Checking for block:" + (i + 1)); while (true) { // Loop to check for block i (usually when 0 is done all will be done blocks = namenode.GetBlockLocations(filename, 0, long.MaxValue); NUnit.Framework.Assert.AreEqual(numBlocks, blocks.LocatedBlockCount()); LocatedBlock block = blocks.Get(i); int actual = block.GetLocations().Length; if (actual == expected) { Log.Info("Got enough replicas for " + (i + 1) + "th block " + block.GetBlock() + ", got " + actual + "."); break; } Log.Info("Not enough replicas for " + (i + 1) + "th block " + block.GetBlock() + " yet. Expecting " + expected + ", got " + actual + "."); if (maxWaitSec > 0 && (Time.MonotonicNow() - start) > (maxWaitSec * 1000)) { throw new IOException("Timedout while waiting for all blocks to " + " be replicated for " + filename); } try { Sharpen.Thread.Sleep(500); } catch (Exception) { } } } }
/// <exception cref="System.IO.IOException"/> public static LocatedBlock GetLastLocatedBlock(ClientProtocol namenode, string src ) { //get block info for the last block LocatedBlocks locations = namenode.GetBlockLocations(src, 0, long.MaxValue); IList <LocatedBlock> blocks = locations.GetLocatedBlocks(); DataNode.Log.Info("blocks.size()=" + blocks.Count); NUnit.Framework.Assert.IsTrue(blocks.Count > 0); return(blocks[blocks.Count - 1]); }
// Waits for all of the blocks to have expected replication /// <exception cref="System.IO.IOException"/> private void WaitForBlockReplication(string filename, ClientProtocol namenode, int expected, long maxWaitSec) { long start = Time.MonotonicNow(); //wait for all the blocks to be replicated; Log.Info("Checking for block replication for " + filename); while (true) { bool replOk = true; LocatedBlocks blocks = namenode.GetBlockLocations(filename, 0, long.MaxValue); for (IEnumerator <LocatedBlock> iter = blocks.GetLocatedBlocks().GetEnumerator(); iter.HasNext();) { LocatedBlock block = iter.Next(); int actual = block.GetLocations().Length; if (actual < expected) { Log.Info("Not enough replicas for " + block.GetBlock() + " yet. Expecting " + expected + ", got " + actual + "."); replOk = false; break; } } if (replOk) { return; } if (maxWaitSec > 0 && (Time.MonotonicNow() - start) > (maxWaitSec * 1000)) { throw new IOException("Timedout while waiting for all blocks to " + " be replicated for " + filename); } try { Sharpen.Thread.Sleep(500); } catch (Exception) { } } }
/* check if there are at least two nodes are on the same rack */ /// <exception cref="System.IO.IOException"/> private void CheckFile(FileSystem fileSys, Path name, int repl) { Configuration conf = fileSys.GetConf(); ClientProtocol namenode = NameNodeProxies.CreateProxy <ClientProtocol>(conf, fileSys .GetUri()).GetProxy(); WaitForBlockReplication(name.ToString(), namenode, Math.Min(numDatanodes, repl), -1); LocatedBlocks locations = namenode.GetBlockLocations(name.ToString(), 0, long.MaxValue ); FileStatus stat = fileSys.GetFileStatus(name); BlockLocation[] blockLocations = fileSys.GetFileBlockLocations(stat, 0L, long.MaxValue ); // verify that rack locations match NUnit.Framework.Assert.IsTrue(blockLocations.Length == locations.LocatedBlockCount ()); for (int i = 0; i < blockLocations.Length; i++) { LocatedBlock blk = locations.Get(i); DatanodeInfo[] datanodes = blk.GetLocations(); string[] topologyPaths = blockLocations[i].GetTopologyPaths(); NUnit.Framework.Assert.IsTrue(topologyPaths.Length == datanodes.Length); for (int j = 0; j < topologyPaths.Length; j++) { bool found = false; for (int k = 0; k < racks.Length; k++) { if (topologyPaths[j].StartsWith(racks[k])) { found = true; break; } } NUnit.Framework.Assert.IsTrue(found); } } bool isOnSameRack = true; bool isNotOnSameRack = true; foreach (LocatedBlock blk_1 in locations.GetLocatedBlocks()) { DatanodeInfo[] datanodes = blk_1.GetLocations(); if (datanodes.Length <= 1) { break; } if (datanodes.Length == 2) { isNotOnSameRack = !(datanodes[0].GetNetworkLocation().Equals(datanodes[1].GetNetworkLocation ())); break; } isOnSameRack = false; isNotOnSameRack = false; for (int i_1 = 0; i_1 < datanodes.Length - 1; i_1++) { Log.Info("datanode " + i_1 + ": " + datanodes[i_1]); bool onRack = false; for (int j = i_1 + 1; j < datanodes.Length; j++) { if (datanodes[i_1].GetNetworkLocation().Equals(datanodes[j].GetNetworkLocation())) { onRack = true; } } if (onRack) { isOnSameRack = true; } if (!onRack) { isNotOnSameRack = true; } if (isOnSameRack && isNotOnSameRack) { break; } } if (!isOnSameRack || !isNotOnSameRack) { break; } } NUnit.Framework.Assert.IsTrue(isOnSameRack); NUnit.Framework.Assert.IsTrue(isNotOnSameRack); }