public virtual void TestRefreshCallQueueProtocol() { RefreshCallQueueProtocolClientSideTranslatorPB translator = (RefreshCallQueueProtocolClientSideTranslatorPB )NameNodeProxies.CreateNonHAProxy <RefreshCallQueueProtocol>(conf, nnAddress, UserGroupInformation .GetCurrentUser(), true).GetProxy(); NUnit.Framework.Assert.IsTrue(translator.IsMethodSupported("refreshCallQueue")); }
public virtual void TestGetUserMappingsProtocol() { GetUserMappingsProtocolClientSideTranslatorPB translator = (GetUserMappingsProtocolClientSideTranslatorPB )NameNodeProxies.CreateNonHAProxy <GetUserMappingsProtocol>(conf, nnAddress, UserGroupInformation .GetCurrentUser(), true).GetProxy(); NUnit.Framework.Assert.IsTrue(translator.IsMethodSupported("getGroupsForUser")); }
public virtual void TestClientNamenodeProtocol() { ClientProtocol cp = NameNodeProxies.CreateNonHAProxy <ClientProtocol>(conf, nnAddress , UserGroupInformation.GetCurrentUser(), true).GetProxy(); RpcClientUtil.IsMethodSupported(cp, typeof(ClientNamenodeProtocolPB), RPC.RpcKind .RpcProtocolBuffer, RPC.GetProtocolVersion(typeof(ClientNamenodeProtocolPB)), "mkdirs" ); }
public virtual void TesJournalProtocol() { JournalProtocolTranslatorPB translator = (JournalProtocolTranslatorPB)NameNodeProxies .CreateNonHAProxy <JournalProtocol>(conf, nnAddress, UserGroupInformation.GetCurrentUser (), true).GetProxy(); //Nameode doesn't implement JournalProtocol NUnit.Framework.Assert.IsFalse(translator.IsMethodSupported("startLogSegment")); }
public DummyLegacyFailoverProxyProvider(Configuration conf, URI uri, Type xface) { try { this.proxy = NameNodeProxies.CreateNonHAProxy(conf, NameNode.GetAddress(uri), xface , UserGroupInformation.GetCurrentUser(), false).GetProxy(); this.xface = xface; } catch (IOException) { } }
/// <summary> /// Check whether logical URI is needed for the namenode and /// the corresponding failover proxy provider in the config. /// </summary> /// <param name="conf">Configuration</param> /// <param name="nameNodeUri">The URI of namenode</param> /// <returns>true if logical URI is needed. false, if not needed.</returns> /// <exception cref="System.IO.IOException">most likely due to misconfiguration.</exception> public static bool UseLogicalUri(Configuration conf, URI nameNodeUri) { // Create the proxy provider. Actual proxy is not created. AbstractNNFailoverProxyProvider <ClientProtocol> provider = NameNodeProxies.CreateFailoverProxyProvider <ClientProtocol>(conf, nameNodeUri, false, null); // No need to use logical URI since failover is not configured. if (provider == null) { return(false); } // Check whether the failover proxy provider uses logical URI. return(provider.UseLogicalURI()); }
public virtual void TestNamenodeProtocol() { NamenodeProtocol np = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nnAddress , UserGroupInformation.GetCurrentUser(), true).GetProxy(); bool exists = RpcClientUtil.IsMethodSupported(np, typeof(NamenodeProtocolPB), RPC.RpcKind .RpcProtocolBuffer, RPC.GetProtocolVersion(typeof(NamenodeProtocolPB)), "rollEditLog" ); NUnit.Framework.Assert.IsTrue(exists); exists = RpcClientUtil.IsMethodSupported(np, typeof(NamenodeProtocolPB), RPC.RpcKind .RpcProtocolBuffer, RPC.GetProtocolVersion(typeof(NamenodeProtocolPB)), "bogusMethod" ); NUnit.Framework.Assert.IsFalse(exists); }
/// <summary>Get an RPC proxy for each NN in an HA nameservice.</summary> /// <remarks> /// Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC /// call should be made on every NN in an HA nameservice, not just the active. /// </remarks> /// <param name="conf">configuration</param> /// <param name="nsId">the nameservice to get all of the proxies for.</param> /// <param name="xface">the protocol class.</param> /// <returns>a list of RPC proxies for each NN in the nameservice.</returns> /// <exception cref="System.IO.IOException">in the event of error.</exception> public static IList <NameNodeProxies.ProxyAndInfo <T> > GetProxiesForAllNameNodesInNameservice <T>(Configuration conf, string nsId) { System.Type xface = typeof(T); IDictionary <string, IPEndPoint> nnAddresses = DFSUtil.GetRpcAddressesForNameserviceId (conf, nsId, null); IList <NameNodeProxies.ProxyAndInfo <T> > proxies = new AList <NameNodeProxies.ProxyAndInfo <T> >(nnAddresses.Count); foreach (IPEndPoint nnAddress in nnAddresses.Values) { NameNodeProxies.ProxyAndInfo <T> proxyInfo = null; proxyInfo = NameNodeProxies.CreateNonHAProxy(conf, nnAddress, xface, UserGroupInformation .GetCurrentUser(), false); proxies.AddItem(proxyInfo); } return(proxies); }
/* check if there are at least two nodes are on the same rack */ /// <exception cref="System.IO.IOException"/> private void CheckFile(FileSystem fileSys, Path name, int repl) { Configuration conf = fileSys.GetConf(); ClientProtocol namenode = NameNodeProxies.CreateProxy <ClientProtocol>(conf, fileSys .GetUri()).GetProxy(); WaitForBlockReplication(name.ToString(), namenode, Math.Min(numDatanodes, repl), -1); LocatedBlocks locations = namenode.GetBlockLocations(name.ToString(), 0, long.MaxValue ); FileStatus stat = fileSys.GetFileStatus(name); BlockLocation[] blockLocations = fileSys.GetFileBlockLocations(stat, 0L, long.MaxValue ); // verify that rack locations match NUnit.Framework.Assert.IsTrue(blockLocations.Length == locations.LocatedBlockCount ()); for (int i = 0; i < blockLocations.Length; i++) { LocatedBlock blk = locations.Get(i); DatanodeInfo[] datanodes = blk.GetLocations(); string[] topologyPaths = blockLocations[i].GetTopologyPaths(); NUnit.Framework.Assert.IsTrue(topologyPaths.Length == datanodes.Length); for (int j = 0; j < topologyPaths.Length; j++) { bool found = false; for (int k = 0; k < racks.Length; k++) { if (topologyPaths[j].StartsWith(racks[k])) { found = true; break; } } NUnit.Framework.Assert.IsTrue(found); } } bool isOnSameRack = true; bool isNotOnSameRack = true; foreach (LocatedBlock blk_1 in locations.GetLocatedBlocks()) { DatanodeInfo[] datanodes = blk_1.GetLocations(); if (datanodes.Length <= 1) { break; } if (datanodes.Length == 2) { isNotOnSameRack = !(datanodes[0].GetNetworkLocation().Equals(datanodes[1].GetNetworkLocation ())); break; } isOnSameRack = false; isNotOnSameRack = false; for (int i_1 = 0; i_1 < datanodes.Length - 1; i_1++) { Log.Info("datanode " + i_1 + ": " + datanodes[i_1]); bool onRack = false; for (int j = i_1 + 1; j < datanodes.Length; j++) { if (datanodes[i_1].GetNetworkLocation().Equals(datanodes[j].GetNetworkLocation())) { onRack = true; } } if (onRack) { isOnSameRack = true; } if (!onRack) { isNotOnSameRack = true; } if (isOnSameRack && isNotOnSameRack) { break; } } if (!isOnSameRack || !isNotOnSameRack) { break; } } NUnit.Framework.Assert.IsTrue(isOnSameRack); NUnit.Framework.Assert.IsTrue(isNotOnSameRack); }
public virtual void TestGetBlocks() { Configuration Conf = new HdfsConfiguration(); short ReplicationFactor = (short)2; int DefaultBlockSize = 1024; Random r = new Random(); Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize); MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(ReplicationFactor ).Build(); try { cluster.WaitActive(); // create a file with two blocks FileSystem fs = cluster.GetFileSystem(); FSDataOutputStream @out = fs.Create(new Path("/tmp.txt"), ReplicationFactor); byte[] data = new byte[1024]; long fileLen = 2 * DefaultBlockSize; long bytesToWrite = fileLen; while (bytesToWrite > 0) { r.NextBytes(data); int bytesToWriteNext = (1024 < bytesToWrite) ? 1024 : (int)bytesToWrite; @out.Write(data, 0, bytesToWriteNext); bytesToWrite -= bytesToWriteNext; } @out.Close(); // get blocks & data nodes IList <LocatedBlock> locatedBlocks; DatanodeInfo[] dataNodes = null; bool notWritten; do { DFSClient dfsclient = new DFSClient(NameNode.GetAddress(Conf), Conf); locatedBlocks = dfsclient.GetNamenode().GetBlockLocations("/tmp.txt", 0, fileLen) .GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(2, locatedBlocks.Count); notWritten = false; for (int i = 0; i < 2; i++) { dataNodes = locatedBlocks[i].GetLocations(); if (dataNodes.Length != ReplicationFactor) { notWritten = true; try { Sharpen.Thread.Sleep(10); } catch (Exception) { } break; } } }while (notWritten); // get RPC client to namenode IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort()); NamenodeProtocol namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(Conf, NameNode .GetUri(addr)).GetProxy(); // get blocks of size fileLen from dataNodes[0] BlocksWithLocations.BlockWithLocations[] locs; locs = namenode.GetBlocks(dataNodes[0], fileLen).GetBlocks(); NUnit.Framework.Assert.AreEqual(locs.Length, 2); NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2); NUnit.Framework.Assert.AreEqual(locs[1].GetStorageIDs().Length, 2); // get blocks of size BlockSize from dataNodes[0] locs = namenode.GetBlocks(dataNodes[0], DefaultBlockSize).GetBlocks(); NUnit.Framework.Assert.AreEqual(locs.Length, 1); NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2); // get blocks of size 1 from dataNodes[0] locs = namenode.GetBlocks(dataNodes[0], 1).GetBlocks(); NUnit.Framework.Assert.AreEqual(locs.Length, 1); NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2); // get blocks of size 0 from dataNodes[0] GetBlocksWithException(namenode, dataNodes[0], 0); // get blocks of size -1 from dataNodes[0] GetBlocksWithException(namenode, dataNodes[0], -1); // get blocks of size BlockSize from a non-existent datanode DatanodeInfo info = DFSTestUtil.GetDatanodeInfo("1.2.3.4"); GetBlocksWithException(namenode, info, 2); } finally { cluster.Shutdown(); } }