/// <exception cref="System.IO.IOException"/> internal EditLogBackupOutputStream(NamenodeRegistration bnReg, JournalInfo journalInfo ) : base() { // RPC proxy to backup node // backup node registration // active node registration // serialized output sent to backup node // backup node // active name-node this.bnRegistration = bnReg; this.journalInfo = journalInfo; IPEndPoint bnAddress = NetUtils.CreateSocketAddr(bnRegistration.GetAddress()); try { this.backupNode = NameNodeProxies.CreateNonHAProxy <JournalProtocol>(new HdfsConfiguration (), bnAddress, UserGroupInformation.GetCurrentUser(), true).GetProxy(); } catch (IOException e) { Storage.Log.Error("Error connecting to: " + bnAddress, e); throw; } this.doubleBuf = new EditsDoubleBuffer(DefaultBufferSize); this.@out = new DataOutputBuffer(DefaultBufferSize); }
/// <summary> /// Create a 4 nodes cluster: 2 nodes (n0, n1) in RACK0/NODEGROUP0, 1 node (n2) /// in RACK1/NODEGROUP1 and 1 node (n3) in RACK1/NODEGROUP2. /// </summary> /// <remarks> /// Create a 4 nodes cluster: 2 nodes (n0, n1) in RACK0/NODEGROUP0, 1 node (n2) /// in RACK1/NODEGROUP1 and 1 node (n3) in RACK1/NODEGROUP2. Fill the cluster /// to 60% and 3 replicas, so n2 and n3 will have replica for all blocks according /// to replica placement policy with NodeGroup. As a result, n2 and n3 will be /// filled with 80% (60% x 4 / 3), and no blocks can be migrated from n2 and n3 /// to n0 or n1 as balancer policy with node group. Thus, we expect the balancer /// to end in 5 iterations without move block process. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestBalancerEndInNoMoveProgress() { Configuration conf = CreateConf(); long[] capacities = new long[] { Capacity, Capacity, Capacity, Capacity }; string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1 }; string[] nodeGroups = new string[] { Nodegroup0, Nodegroup0, Nodegroup1, Nodegroup2 }; int numOfDatanodes = capacities.Length; NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length); NUnit.Framework.Assert.AreEqual(numOfDatanodes, nodeGroups.Length); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities .Length).Racks(racks).SimulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder); try { cluster.WaitActive(); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem( 0).GetUri()).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 60% full long totalUsedSpace = totalCapacity * 6 / 10; TestBalancer.CreateFile(cluster, filePath, totalUsedSpace / 3, (short)(3), 0); // run balancer which can finish in 5 iterations with no block movement. RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity); } finally { cluster.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> public NameNodeConnector(string name, URI nameNodeUri, Path idPath, IList <Path> targetPaths , Configuration conf, int maxNotChangedIterations) { this.nameNodeUri = nameNodeUri; this.idPath = idPath; this.targetPaths = targetPaths == null || targetPaths.IsEmpty() ? Arrays.AsList(new Path("/")) : targetPaths; this.maxNotChangedIterations = maxNotChangedIterations; this.namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(conf, nameNodeUri). GetProxy(); this.client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, nameNodeUri, fallbackToSimpleAuth ).GetProxy(); this.fs = (DistributedFileSystem)FileSystem.Get(nameNodeUri, conf); NamespaceInfo namespaceinfo = namenode.VersionRequest(); this.blockpoolID = namespaceinfo.GetBlockPoolID(); FsServerDefaults defaults = fs.GetServerDefaults(new Path("/")); this.keyManager = new KeyManager(blockpoolID, namenode, defaults.GetEncryptDataTransfer (), conf); // if it is for test, we do not create the id file @out = CheckAndMarkRunning(); if (@out == null) { // Exit if there is another one running. throw new IOException("Another " + name + " is running."); } }
/// <summary> /// Test a cluster with even distribution, then a new empty node is added to /// the cluster. /// </summary> /// <remarks> /// Test a cluster with even distribution, then a new empty node is added to /// the cluster. Test start a cluster with specified number of nodes, and fills /// it to be 30% full (with a single file replicated identically to all /// datanodes); It then adds one new empty node and starts balancing. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestBalancerWithHANameNodes() { Configuration conf = new HdfsConfiguration(); TestBalancer.InitConf(conf); long newNodeCapacity = TestBalancer.Capacity; // new node's capacity string newNodeRack = TestBalancer.Rack2; // new node's rack // array of racks for original nodes in cluster string[] racks = new string[] { TestBalancer.Rack0, TestBalancer.Rack1 }; // array of capacities of original nodes in cluster long[] capacities = new long[] { TestBalancer.Capacity, TestBalancer.Capacity }; NUnit.Framework.Assert.AreEqual(capacities.Length, racks.Length); int numOfDatanodes = capacities.Length; MiniDFSNNTopology.NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1"); nn1Conf.SetIpcPort(NameNode.DefaultPort); Configuration copiedConf = new Configuration(conf); cluster = new MiniDFSCluster.Builder(copiedConf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(capacities.Length).Racks(racks).SimulatedCapacities(capacities) .Build(); HATestUtil.SetFailoverConfigurations(cluster, conf); try { cluster.WaitActive(); cluster.TransitionToActive(1); Sharpen.Thread.Sleep(500); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, FileSystem.GetDefaultUri (conf)).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity * 3 / 10; TestBalancer.CreateFile(cluster, TestBalancer.filePath, totalUsedSpace / numOfDatanodes , (short)numOfDatanodes, 1); // start up an empty node with the same capacity and on the same rack cluster.StartDataNodes(conf, 1, true, null, new string[] { newNodeRack }, new long [] { newNodeCapacity }); totalCapacity += newNodeCapacity; TestBalancer.WaitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster); ICollection <URI> namenodes = DFSUtil.GetNsServiceRpcUris(conf); NUnit.Framework.Assert.AreEqual(1, namenodes.Count); NUnit.Framework.Assert.IsTrue(namenodes.Contains(HATestUtil.GetLogicalUri(cluster ))); int r = Org.Apache.Hadoop.Hdfs.Server.Balancer.Balancer.Run(namenodes, Balancer.Parameters .Default, conf); NUnit.Framework.Assert.AreEqual(ExitStatus.Success.GetExitCode(), r); TestBalancer.WaitForBalancer(totalUsedSpace, totalCapacity, client, cluster, Balancer.Parameters .Default); } finally { cluster.Shutdown(); } }
/// <summary> /// Create a cluster with even distribution, and a new empty node is added to /// the cluster, then test rack locality for balancer policy. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestBalancerWithRackLocality() { Configuration conf = CreateConf(); long[] capacities = new long[] { Capacity, Capacity }; string[] racks = new string[] { Rack0, Rack1 }; string[] nodeGroups = new string[] { Nodegroup0, Nodegroup1 }; int numOfDatanodes = capacities.Length; NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities .Length).Racks(racks).SimulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder); try { cluster.WaitActive(); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem( 0).GetUri()).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity * 3 / 10; long length = totalUsedSpace / numOfDatanodes; TestBalancer.CreateFile(cluster, filePath, length, (short)numOfDatanodes, 0); LocatedBlocks lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length ); ICollection <ExtendedBlock> before = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0 ); long newCapacity = Capacity; string newRack = Rack1; string newNodeGroup = Nodegroup2; // start up an empty node with the same capacity and on the same rack cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[] { newCapacity }, new string[] { newNodeGroup }); totalCapacity += newCapacity; // run balancer and validate results RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity); lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length); ICollection <ExtendedBlock> after = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0); NUnit.Framework.Assert.AreEqual(before, after); } finally { cluster.Shutdown(); } }
/// <summary> /// Create a /// <see cref="NameNode"/> /// proxy from the current /// <see cref="Javax.Servlet.ServletContext"/> /// . /// </summary> /// <exception cref="System.IO.IOException"/> protected internal virtual ClientProtocol CreateNameNodeProxy() { ServletContext context = GetServletContext(); // if we are running in the Name Node, use it directly rather than via // rpc NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context); if (nn != null) { return(nn.GetRpcServer()); } IPEndPoint nnAddr = NameNodeHttpServer.GetNameNodeAddressFromContext(context); Configuration conf = new HdfsConfiguration(NameNodeHttpServer.GetConfFromContext( context)); return(NameNodeProxies.CreateProxy <ClientProtocol>(conf, NameNode.GetUri(nnAddr)) .GetProxy()); }
/// <summary> /// Create a cluster with even distribution, and a new empty node is added to /// the cluster, then test node-group locality for balancer policy. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestBalancerWithNodeGroup() { Configuration conf = CreateConf(); long[] capacities = new long[] { Capacity, Capacity, Capacity, Capacity }; string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1 }; string[] nodeGroups = new string[] { Nodegroup0, Nodegroup0, Nodegroup1, Nodegroup2 }; int numOfDatanodes = capacities.Length; NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length); NUnit.Framework.Assert.AreEqual(numOfDatanodes, nodeGroups.Length); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities .Length).Racks(racks).SimulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder); try { cluster.WaitActive(); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem( 0).GetUri()).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 20% full long totalUsedSpace = totalCapacity * 2 / 10; TestBalancer.CreateFile(cluster, filePath, totalUsedSpace / (numOfDatanodes / 2), (short)(numOfDatanodes / 2), 0); long newCapacity = Capacity; string newRack = Rack1; string newNodeGroup = Nodegroup2; // start up an empty node with the same capacity and on NODEGROUP2 cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[] { newCapacity }, new string[] { newNodeGroup }); totalCapacity += newCapacity; // run balancer and validate results RunBalancer(conf, totalUsedSpace, totalCapacity); } finally { cluster.Shutdown(); } }
public override FailoverProxyProvider.ProxyInfo <T> GetProxy() { lock (this) { // Create a non-ha proxy if not already created. if (nnProxyInfo == null) { try { // Create a proxy that is not wrapped in RetryProxy IPEndPoint nnAddr = NameNode.GetAddress(nameNodeUri); nnProxyInfo = new FailoverProxyProvider.ProxyInfo <T>(NameNodeProxies.CreateNonHAProxy (conf, nnAddr, xface, UserGroupInformation.GetCurrentUser(), false).GetProxy(), nnAddr.ToString()); } catch (IOException ioe) { throw new RuntimeException(ioe); } } return(nnProxyInfo); } }
/// <summary>Lazily initialize the RPC proxy object.</summary> public override FailoverProxyProvider.ProxyInfo <T> GetProxy() { lock (this) { ConfiguredFailoverProxyProvider.AddressRpcProxyPair <T> current = proxies[currentProxyIndex ]; if (current.namenode == null) { try { current.namenode = NameNodeProxies.CreateNonHAProxy(conf, current.address, xface, ugi, false, fallbackToSimpleAuth).GetProxy(); } catch (IOException e) { Log.Error("Failed to create RPC proxy to NameNode", e); throw new RuntimeException(e); } } return(new FailoverProxyProvider.ProxyInfo <T>(current.namenode, current.address.ToString ())); } }
/// <exception cref="System.IO.IOException"/> private NamespaceInfo Handshake(Configuration conf) { // connect to name node IPEndPoint nnAddress = NameNode.GetServiceAddress(conf, true); this.namenode = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nnAddress , UserGroupInformation.GetCurrentUser(), true).GetProxy(); this.nnRpcAddress = NetUtils.GetHostPortString(nnAddress); this.nnHttpAddress = DFSUtil.GetInfoServer(nnAddress, conf, DFSUtil.GetHttpClientScheme (conf)).ToURL(); // get version and id info from the name-node NamespaceInfo nsInfo = null; while (!IsStopRequested()) { try { nsInfo = Handshake(namenode); break; } catch (SocketTimeoutException e) { // name-node is busy Log.Info("Problem connecting to server: " + nnAddress); try { Sharpen.Thread.Sleep(1000); } catch (Exception) { Log.Warn("Encountered exception ", e); } } } return(nsInfo); }
/// <exception cref="System.IO.IOException"/> private NamenodeProtocol CreateNNProtocolProxy() { return(NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(GetConf(), otherIpcAddr , UserGroupInformation.GetLoginUser(), true).GetProxy()); }
/// <exception cref="System.IO.IOException"/> protected override GetUserMappingsProtocol GetUgmProtocol() { return(NameNodeProxies.CreateProxy <GetUserMappingsProtocol>(GetConf(), FileSystem .GetDefaultUri(GetConf())).GetProxy()); }
/// <summary>Initialize SecondaryNameNode.</summary> /// <exception cref="System.IO.IOException"/> private void Initialize(Configuration conf, SecondaryNameNode.CommandLineOpts commandLineOpts ) { IPEndPoint infoSocAddr = GetHttpAddress(conf); string infoBindAddress = infoSocAddr.GetHostName(); UserGroupInformation.SetConfiguration(conf); if (UserGroupInformation.IsSecurityEnabled()) { SecurityUtil.Login(conf, DFSConfigKeys.DfsSecondaryNamenodeKeytabFileKey, DFSConfigKeys .DfsSecondaryNamenodeKerberosPrincipalKey, infoBindAddress); } // initiate Java VM metrics DefaultMetricsSystem.Initialize("SecondaryNameNode"); JvmMetrics.Create("SecondaryNameNode", conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey ), DefaultMetricsSystem.Instance()); // Create connection to the namenode. shouldRun = true; nameNodeAddr = NameNode.GetServiceAddress(conf, true); this.conf = conf; this.namenode = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nameNodeAddr , UserGroupInformation.GetCurrentUser(), true).GetProxy(); // initialize checkpoint directories fsName = GetInfoServer(); checkpointDirs = FSImage.GetCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary"); checkpointEditsDirs = FSImage.GetCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary" ); checkpointImage = new SecondaryNameNode.CheckpointStorage(conf, checkpointDirs, checkpointEditsDirs ); checkpointImage.RecoverCreate(commandLineOpts.ShouldFormat()); checkpointImage.DeleteTempEdits(); namesystem = new FSNamesystem(conf, checkpointImage, true); // Disable quota checks namesystem.dir.DisableQuotaChecks(); // Initialize other scheduling parameters from the configuration checkpointConf = new CheckpointConf(conf); IPEndPoint httpAddr = infoSocAddr; string httpsAddrString = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressKey , DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressDefault); IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString); HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr , httpsAddr, "secondary", DFSConfigKeys.DfsSecondaryNamenodeKerberosInternalSpnegoPrincipalKey , DFSConfigKeys.DfsSecondaryNamenodeKeytabFileKey); nameNodeStatusBeanName = MBeans.Register("SecondaryNameNode", "SecondaryNameNodeInfo" , this); infoServer = builder.Build(); infoServer.SetAttribute("secondary.name.node", this); infoServer.SetAttribute("name.system.image", checkpointImage); infoServer.SetAttribute(JspHelper.CurrentConf, conf); infoServer.AddInternalServlet("imagetransfer", ImageServlet.PathSpec, typeof(ImageServlet ), true); infoServer.Start(); Log.Info("Web server init done"); HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf); int connIdx = 0; if (policy.IsHttpEnabled()) { IPEndPoint httpAddress = infoServer.GetConnectorAddress(connIdx++); conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey, NetUtils.GetHostPortString (httpAddress)); } if (policy.IsHttpsEnabled()) { IPEndPoint httpsAddress = infoServer.GetConnectorAddress(connIdx); conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressKey, NetUtils.GetHostPortString (httpsAddress)); } legacyOivImageDir = conf.Get(DFSConfigKeys.DfsNamenodeLegacyOivImageDirKey); Log.Info("Checkpoint Period :" + checkpointConf.GetPeriod() + " secs " + "(" + checkpointConf.GetPeriod() / 60 + " min)"); Log.Info("Log Size Trigger :" + checkpointConf.GetTxnCount() + " txns"); }