/// <exception cref="System.IO.IOException"/> private void ParseConfAndFindOtherNN() { Configuration conf = GetConf(); nsId = DFSUtil.GetNamenodeNameServiceId(conf); if (!HAUtil.IsHAEnabled(conf, nsId)) { throw new HadoopIllegalArgumentException("HA is not enabled for this namenode."); } nnId = HAUtil.GetNameNodeId(conf, nsId); NameNode.InitializeGenericKeys(conf, nsId, nnId); if (!HAUtil.UsesSharedEditsDir(conf)) { throw new HadoopIllegalArgumentException("Shared edits storage is not enabled for this namenode." ); } Configuration otherNode = HAUtil.GetConfForOtherNode(conf); otherNNId = HAUtil.GetNameNodeId(otherNode, nsId); otherIpcAddr = NameNode.GetServiceAddress(otherNode, true); Preconditions.CheckArgument(otherIpcAddr.Port != 0 && !otherIpcAddr.Address.IsAnyLocalAddress (), "Could not determine valid IPC address for other NameNode (%s)" + ", got: %s" , otherNNId, otherIpcAddr); string scheme = DFSUtil.GetHttpClientScheme(conf); otherHttpAddr = DFSUtil.GetInfoServerWithDefaultHost(otherIpcAddr.GetHostName(), otherNode, scheme).ToURL(); dirsToFormat = FSNamesystem.GetNamespaceDirs(conf); editUrisToFormat = FSNamesystem.GetNamespaceEditsDirs(conf, false); sharedEditsUris = FSNamesystem.GetSharedEditsDirs(conf); }
public virtual void TestClientSideException() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); NNStorage mockStorage = Org.Mockito.Mockito.Mock <NNStorage>(); IList <FilePath> localPath = Collections.SingletonList(new FilePath("/xxxxx-does-not-exist/blah" )); try { Uri fsName = DFSUtil.GetInfoServer(cluster.GetNameNode().GetServiceRpcAddress(), conf, DFSUtil.GetHttpClientScheme(conf)).ToURL(); string id = "getimage=1&txid=0"; TransferFsImage.GetFileClient(fsName, id, localPath, mockStorage, false); NUnit.Framework.Assert.Fail("Didn't get an exception!"); } catch (IOException ioe) { Org.Mockito.Mockito.Verify(mockStorage).ReportErrorOnFile(localPath[0]); NUnit.Framework.Assert.IsTrue("Unexpected exception: " + StringUtils.StringifyException (ioe), ioe.Message.Contains("Unable to download to any storage")); } finally { cluster.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> private Uri GetHttpAddress(Configuration conf) { string scheme = DFSUtil.GetHttpClientScheme(conf); string defaultHost = NameNode.GetServiceAddress(conf, true).GetHostName(); URI addr = DFSUtil.GetInfoServerWithDefaultHost(defaultHost, conf, scheme); return(addr.ToURL()); }
/// <summary>Return the URI that locates the HTTP server.</summary> internal virtual URI GetServerURI() { // getHttpClientScheme() only returns https for HTTPS_ONLY policy. This // matches the behavior that the first connector is a HTTPS connector only // for HTTPS_ONLY policy. IPEndPoint addr = httpServer.GetConnectorAddress(0); return(URI.Create(DFSUtil.GetHttpClientScheme(conf) + "://" + NetUtils.GetHostPortString (addr))); }
/// <summary> /// Derive the namenode http address from the current file system, /// either default or as set by "-fs" in the generic options. /// </summary> /// <returns>Returns http address or null if failure.</returns> /// <exception cref="System.IO.IOException">if we can't determine the active NN address /// </exception> private URI GetCurrentNamenodeAddress(Path target) { //String nnAddress = null; Configuration conf = GetConf(); //get the filesystem object to verify it is an HDFS system FileSystem fs = target.GetFileSystem(conf); if (!(fs is DistributedFileSystem)) { System.Console.Error.WriteLine("FileSystem is " + fs.GetUri()); return(null); } return(DFSUtil.GetInfoServer(HAUtil.GetAddressOfActive(fs), conf, DFSUtil.GetHttpClientScheme (conf))); }
private Uri GetImageListenAddress() { IPEndPoint httpSocAddr = backupNode.GetHttpAddress(); int httpPort = httpSocAddr.Port; try { return(new Uri(DFSUtil.GetHttpClientScheme(conf) + "://" + infoBindAddress + ":" + httpPort)); } catch (UriFormatException e) { // Unreachable throw new RuntimeException(e); } }
/// <summary>Returns the Jetty server that the Namenode is listening on.</summary> /// <exception cref="System.IO.IOException"/> private Uri GetInfoServer() { URI fsName = FileSystem.GetDefaultUri(conf); if (!Sharpen.Runtime.EqualsIgnoreCase(HdfsConstants.HdfsUriScheme, fsName.GetScheme ())) { throw new IOException("This is not a DFS"); } string scheme = DFSUtil.GetHttpClientScheme(conf); URI address = DFSUtil.GetInfoServerWithDefaultHost(fsName.GetHost(), conf, scheme ); Log.Debug("Will connect to NameNode at " + address); return(address.ToURL()); }
public virtual void TestClientSideExceptionOnJustOneDir() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); NNStorage mockStorage = Org.Mockito.Mockito.Mock <NNStorage>(); IList <FilePath> localPaths = ImmutableList.Of(new FilePath("/xxxxx-does-not-exist/blah" ), new FilePath(TestDir, "testfile")); try { Uri fsName = DFSUtil.GetInfoServer(cluster.GetNameNode().GetServiceRpcAddress(), conf, DFSUtil.GetHttpClientScheme(conf)).ToURL(); string id = "getimage=1&txid=0"; TransferFsImage.GetFileClient(fsName, id, localPaths, mockStorage, false); Org.Mockito.Mockito.Verify(mockStorage).ReportErrorOnFile(localPaths[0]); NUnit.Framework.Assert.IsTrue("The valid local file should get saved properly", localPaths [1].Length() > 0); } finally { cluster.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> private NamespaceInfo Handshake(Configuration conf) { // connect to name node IPEndPoint nnAddress = NameNode.GetServiceAddress(conf, true); this.namenode = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nnAddress , UserGroupInformation.GetCurrentUser(), true).GetProxy(); this.nnRpcAddress = NetUtils.GetHostPortString(nnAddress); this.nnHttpAddress = DFSUtil.GetInfoServer(nnAddress, conf, DFSUtil.GetHttpClientScheme (conf)).ToURL(); // get version and id info from the name-node NamespaceInfo nsInfo = null; while (!IsStopRequested()) { try { nsInfo = Handshake(namenode); break; } catch (SocketTimeoutException e) { // name-node is busy Log.Info("Problem connecting to server: " + nnAddress); try { Sharpen.Thread.Sleep(1000); } catch (Exception) { Log.Warn("Encountered exception ", e); } } } return(nsInfo); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Javax.Management.MalformedObjectNameException"/> internal NamenodeMXBeanHelper(IPEndPoint addr, Configuration conf) { this.host = addr.GetHostName(); this.httpAddress = DFSUtil.GetInfoServer(addr, conf, DFSUtil.GetHttpClientScheme( conf)); }