/// <exception cref="System.IO.IOException"/>
internal EditLogBackupOutputStream(NamenodeRegistration bnReg, JournalInfo journalInfo
)
: base()
{
// RPC proxy to backup node
// backup node registration
// active node registration
// serialized output sent to backup node
// backup node
// active name-node
this.bnRegistration = bnReg;
this.journalInfo = journalInfo;
IPEndPoint bnAddress = NetUtils.CreateSocketAddr(bnRegistration.GetAddress());
try
{
this.backupNode = NameNodeProxies.CreateNonHAProxy <JournalProtocol>(new HdfsConfiguration
(), bnAddress, UserGroupInformation.GetCurrentUser(), true).GetProxy();
}
catch (IOException e)
{
Storage.Log.Error("Error connecting to: " + bnAddress, e);
throw;
}
this.doubleBuf = new EditsDoubleBuffer(DefaultBufferSize);
this.@out = new DataOutputBuffer(DefaultBufferSize);
}
/// <summary>
/// Create a 4 nodes cluster: 2 nodes (n0, n1) in RACK0/NODEGROUP0, 1 node (n2)
/// in RACK1/NODEGROUP1 and 1 node (n3) in RACK1/NODEGROUP2.
/// </summary>
/// <remarks>
/// Create a 4 nodes cluster: 2 nodes (n0, n1) in RACK0/NODEGROUP0, 1 node (n2)
/// in RACK1/NODEGROUP1 and 1 node (n3) in RACK1/NODEGROUP2. Fill the cluster
/// to 60% and 3 replicas, so n2 and n3 will have replica for all blocks according
/// to replica placement policy with NodeGroup. As a result, n2 and n3 will be
/// filled with 80% (60% x 4 / 3), and no blocks can be migrated from n2 and n3
/// to n0 or n1 as balancer policy with node group. Thus, we expect the balancer
/// to end in 5 iterations without move block process.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestBalancerEndInNoMoveProgress()
{
Configuration conf = CreateConf();
long[] capacities = new long[] { Capacity, Capacity, Capacity, Capacity };
string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1 };
string[] nodeGroups = new string[] { Nodegroup0, Nodegroup0, Nodegroup1, Nodegroup2 };
int numOfDatanodes = capacities.Length;
NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length);
NUnit.Framework.Assert.AreEqual(numOfDatanodes, nodeGroups.Length);
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities
.Length).Racks(racks).SimulatedCapacities(capacities);
MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups);
cluster = new MiniDFSClusterWithNodeGroup(builder);
try
{
cluster.WaitActive();
client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem(
0).GetUri()).GetProxy();
long totalCapacity = TestBalancer.Sum(capacities);
// fill up the cluster to be 60% full
long totalUsedSpace = totalCapacity * 6 / 10;
TestBalancer.CreateFile(cluster, filePath, totalUsedSpace / 3, (short)(3), 0);
// run balancer which can finish in 5 iterations with no block movement.
RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity);
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
public NameNodeConnector(string name, URI nameNodeUri, Path idPath, IList <Path> targetPaths
, Configuration conf, int maxNotChangedIterations)
{
this.nameNodeUri = nameNodeUri;
this.idPath = idPath;
this.targetPaths = targetPaths == null || targetPaths.IsEmpty() ? Arrays.AsList(new
Path("/")) : targetPaths;
this.maxNotChangedIterations = maxNotChangedIterations;
this.namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(conf, nameNodeUri).
GetProxy();
this.client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, nameNodeUri, fallbackToSimpleAuth
).GetProxy();
this.fs = (DistributedFileSystem)FileSystem.Get(nameNodeUri, conf);
NamespaceInfo namespaceinfo = namenode.VersionRequest();
this.blockpoolID = namespaceinfo.GetBlockPoolID();
FsServerDefaults defaults = fs.GetServerDefaults(new Path("/"));
this.keyManager = new KeyManager(blockpoolID, namenode, defaults.GetEncryptDataTransfer
(), conf);
// if it is for test, we do not create the id file
@out = CheckAndMarkRunning();
if (@out == null)
{
// Exit if there is another one running.
throw new IOException("Another " + name + " is running.");
}
}
/// <summary>
/// Test a cluster with even distribution, then a new empty node is added to
/// the cluster.
/// </summary>
/// <remarks>
/// Test a cluster with even distribution, then a new empty node is added to
/// the cluster. Test start a cluster with specified number of nodes, and fills
/// it to be 30% full (with a single file replicated identically to all
/// datanodes); It then adds one new empty node and starts balancing.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestBalancerWithHANameNodes()
{
Configuration conf = new HdfsConfiguration();
TestBalancer.InitConf(conf);
long newNodeCapacity = TestBalancer.Capacity;
// new node's capacity
string newNodeRack = TestBalancer.Rack2;
// new node's rack
// array of racks for original nodes in cluster
string[] racks = new string[] { TestBalancer.Rack0, TestBalancer.Rack1 };
// array of capacities of original nodes in cluster
long[] capacities = new long[] { TestBalancer.Capacity, TestBalancer.Capacity };
NUnit.Framework.Assert.AreEqual(capacities.Length, racks.Length);
int numOfDatanodes = capacities.Length;
MiniDFSNNTopology.NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1");
nn1Conf.SetIpcPort(NameNode.DefaultPort);
Configuration copiedConf = new Configuration(conf);
cluster = new MiniDFSCluster.Builder(copiedConf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(capacities.Length).Racks(racks).SimulatedCapacities(capacities)
.Build();
HATestUtil.SetFailoverConfigurations(cluster, conf);
try
{
cluster.WaitActive();
cluster.TransitionToActive(1);
Sharpen.Thread.Sleep(500);
client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, FileSystem.GetDefaultUri
(conf)).GetProxy();
long totalCapacity = TestBalancer.Sum(capacities);
// fill up the cluster to be 30% full
long totalUsedSpace = totalCapacity * 3 / 10;
TestBalancer.CreateFile(cluster, TestBalancer.filePath, totalUsedSpace / numOfDatanodes
, (short)numOfDatanodes, 1);
// start up an empty node with the same capacity and on the same rack
cluster.StartDataNodes(conf, 1, true, null, new string[] { newNodeRack }, new long
[] { newNodeCapacity });
totalCapacity += newNodeCapacity;
TestBalancer.WaitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
ICollection <URI> namenodes = DFSUtil.GetNsServiceRpcUris(conf);
NUnit.Framework.Assert.AreEqual(1, namenodes.Count);
NUnit.Framework.Assert.IsTrue(namenodes.Contains(HATestUtil.GetLogicalUri(cluster
)));
int r = Org.Apache.Hadoop.Hdfs.Server.Balancer.Balancer.Run(namenodes, Balancer.Parameters
.Default, conf);
NUnit.Framework.Assert.AreEqual(ExitStatus.Success.GetExitCode(), r);
TestBalancer.WaitForBalancer(totalUsedSpace, totalCapacity, client, cluster, Balancer.Parameters
.Default);
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Create a cluster with even distribution, and a new empty node is added to
/// the cluster, then test rack locality for balancer policy.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestBalancerWithRackLocality()
{
Configuration conf = CreateConf();
long[] capacities = new long[] { Capacity, Capacity };
string[] racks = new string[] { Rack0, Rack1 };
string[] nodeGroups = new string[] { Nodegroup0, Nodegroup1 };
int numOfDatanodes = capacities.Length;
NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length);
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities
.Length).Racks(racks).SimulatedCapacities(capacities);
MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups);
cluster = new MiniDFSClusterWithNodeGroup(builder);
try
{
cluster.WaitActive();
client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem(
0).GetUri()).GetProxy();
long totalCapacity = TestBalancer.Sum(capacities);
// fill up the cluster to be 30% full
long totalUsedSpace = totalCapacity * 3 / 10;
long length = totalUsedSpace / numOfDatanodes;
TestBalancer.CreateFile(cluster, filePath, length, (short)numOfDatanodes, 0);
LocatedBlocks lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length
);
ICollection <ExtendedBlock> before = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0
);
long newCapacity = Capacity;
string newRack = Rack1;
string newNodeGroup = Nodegroup2;
// start up an empty node with the same capacity and on the same rack
cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[]
{ newCapacity }, new string[] { newNodeGroup });
totalCapacity += newCapacity;
// run balancer and validate results
RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity);
lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length);
ICollection <ExtendedBlock> after = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0);
NUnit.Framework.Assert.AreEqual(before, after);
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Create a
/// <see cref="NameNode"/>
/// proxy from the current
/// <see cref="Javax.Servlet.ServletContext"/>
/// .
/// </summary>
/// <exception cref="System.IO.IOException"/>
protected internal virtual ClientProtocol CreateNameNodeProxy()
{
ServletContext context = GetServletContext();
// if we are running in the Name Node, use it directly rather than via
// rpc
NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context);
if (nn != null)
{
return(nn.GetRpcServer());
}
IPEndPoint nnAddr = NameNodeHttpServer.GetNameNodeAddressFromContext(context);
Configuration conf = new HdfsConfiguration(NameNodeHttpServer.GetConfFromContext(
context));
return(NameNodeProxies.CreateProxy <ClientProtocol>(conf, NameNode.GetUri(nnAddr))
.GetProxy());
}
/// <summary>
/// Create a cluster with even distribution, and a new empty node is added to
/// the cluster, then test node-group locality for balancer policy.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestBalancerWithNodeGroup()
{
Configuration conf = CreateConf();
long[] capacities = new long[] { Capacity, Capacity, Capacity, Capacity };
string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1 };
string[] nodeGroups = new string[] { Nodegroup0, Nodegroup0, Nodegroup1, Nodegroup2 };
int numOfDatanodes = capacities.Length;
NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length);
NUnit.Framework.Assert.AreEqual(numOfDatanodes, nodeGroups.Length);
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities
.Length).Racks(racks).SimulatedCapacities(capacities);
MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups);
cluster = new MiniDFSClusterWithNodeGroup(builder);
try
{
cluster.WaitActive();
client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem(
0).GetUri()).GetProxy();
long totalCapacity = TestBalancer.Sum(capacities);
// fill up the cluster to be 20% full
long totalUsedSpace = totalCapacity * 2 / 10;
TestBalancer.CreateFile(cluster, filePath, totalUsedSpace / (numOfDatanodes / 2),
(short)(numOfDatanodes / 2), 0);
long newCapacity = Capacity;
string newRack = Rack1;
string newNodeGroup = Nodegroup2;
// start up an empty node with the same capacity and on NODEGROUP2
cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[]
{ newCapacity }, new string[] { newNodeGroup });
totalCapacity += newCapacity;
// run balancer and validate results
RunBalancer(conf, totalUsedSpace, totalCapacity);
}
finally
{
cluster.Shutdown();
}
}
public override FailoverProxyProvider.ProxyInfo <T> GetProxy()
{
lock (this)
{
// Create a non-ha proxy if not already created.
if (nnProxyInfo == null)
{
try
{
// Create a proxy that is not wrapped in RetryProxy
IPEndPoint nnAddr = NameNode.GetAddress(nameNodeUri);
nnProxyInfo = new FailoverProxyProvider.ProxyInfo <T>(NameNodeProxies.CreateNonHAProxy
(conf, nnAddr, xface, UserGroupInformation.GetCurrentUser(), false).GetProxy(),
nnAddr.ToString());
}
catch (IOException ioe)
{
throw new RuntimeException(ioe);
}
}
return(nnProxyInfo);
}
}
/// <summary>Lazily initialize the RPC proxy object.</summary>
public override FailoverProxyProvider.ProxyInfo <T> GetProxy()
{
lock (this)
{
ConfiguredFailoverProxyProvider.AddressRpcProxyPair <T> current = proxies[currentProxyIndex
];
if (current.namenode == null)
{
try
{
current.namenode = NameNodeProxies.CreateNonHAProxy(conf, current.address, xface,
ugi, false, fallbackToSimpleAuth).GetProxy();
}
catch (IOException e)
{
Log.Error("Failed to create RPC proxy to NameNode", e);
throw new RuntimeException(e);
}
}
return(new FailoverProxyProvider.ProxyInfo <T>(current.namenode, current.address.ToString
()));
}
}
/// <exception cref="System.IO.IOException"/>
private NamespaceInfo Handshake(Configuration conf)
{
// connect to name node
IPEndPoint nnAddress = NameNode.GetServiceAddress(conf, true);
this.namenode = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nnAddress
, UserGroupInformation.GetCurrentUser(), true).GetProxy();
this.nnRpcAddress = NetUtils.GetHostPortString(nnAddress);
this.nnHttpAddress = DFSUtil.GetInfoServer(nnAddress, conf, DFSUtil.GetHttpClientScheme
(conf)).ToURL();
// get version and id info from the name-node
NamespaceInfo nsInfo = null;
while (!IsStopRequested())
{
try
{
nsInfo = Handshake(namenode);
break;
}
catch (SocketTimeoutException e)
{
// name-node is busy
Log.Info("Problem connecting to server: " + nnAddress);
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
Log.Warn("Encountered exception ", e);
}
}
}
return(nsInfo);
}
/// <exception cref="System.IO.IOException"/>
private NamenodeProtocol CreateNNProtocolProxy()
{
return(NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(GetConf(), otherIpcAddr
, UserGroupInformation.GetLoginUser(), true).GetProxy());
}
/// <exception cref="System.IO.IOException"/>
protected override GetUserMappingsProtocol GetUgmProtocol()
{
return(NameNodeProxies.CreateProxy <GetUserMappingsProtocol>(GetConf(), FileSystem
.GetDefaultUri(GetConf())).GetProxy());
}
/// <summary>Initialize SecondaryNameNode.</summary>
/// <exception cref="System.IO.IOException"/>
private void Initialize(Configuration conf, SecondaryNameNode.CommandLineOpts commandLineOpts
)
{
IPEndPoint infoSocAddr = GetHttpAddress(conf);
string infoBindAddress = infoSocAddr.GetHostName();
UserGroupInformation.SetConfiguration(conf);
if (UserGroupInformation.IsSecurityEnabled())
{
SecurityUtil.Login(conf, DFSConfigKeys.DfsSecondaryNamenodeKeytabFileKey, DFSConfigKeys
.DfsSecondaryNamenodeKerberosPrincipalKey, infoBindAddress);
}
// initiate Java VM metrics
DefaultMetricsSystem.Initialize("SecondaryNameNode");
JvmMetrics.Create("SecondaryNameNode", conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey
), DefaultMetricsSystem.Instance());
// Create connection to the namenode.
shouldRun = true;
nameNodeAddr = NameNode.GetServiceAddress(conf, true);
this.conf = conf;
this.namenode = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nameNodeAddr
, UserGroupInformation.GetCurrentUser(), true).GetProxy();
// initialize checkpoint directories
fsName = GetInfoServer();
checkpointDirs = FSImage.GetCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
checkpointEditsDirs = FSImage.GetCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary"
);
checkpointImage = new SecondaryNameNode.CheckpointStorage(conf, checkpointDirs, checkpointEditsDirs
);
checkpointImage.RecoverCreate(commandLineOpts.ShouldFormat());
checkpointImage.DeleteTempEdits();
namesystem = new FSNamesystem(conf, checkpointImage, true);
// Disable quota checks
namesystem.dir.DisableQuotaChecks();
// Initialize other scheduling parameters from the configuration
checkpointConf = new CheckpointConf(conf);
IPEndPoint httpAddr = infoSocAddr;
string httpsAddrString = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressKey
, DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressDefault);
IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);
HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
, httpsAddr, "secondary", DFSConfigKeys.DfsSecondaryNamenodeKerberosInternalSpnegoPrincipalKey
, DFSConfigKeys.DfsSecondaryNamenodeKeytabFileKey);
nameNodeStatusBeanName = MBeans.Register("SecondaryNameNode", "SecondaryNameNodeInfo"
, this);
infoServer = builder.Build();
infoServer.SetAttribute("secondary.name.node", this);
infoServer.SetAttribute("name.system.image", checkpointImage);
infoServer.SetAttribute(JspHelper.CurrentConf, conf);
infoServer.AddInternalServlet("imagetransfer", ImageServlet.PathSpec, typeof(ImageServlet
), true);
infoServer.Start();
Log.Info("Web server init done");
HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
int connIdx = 0;
if (policy.IsHttpEnabled())
{
IPEndPoint httpAddress = infoServer.GetConnectorAddress(connIdx++);
conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey, NetUtils.GetHostPortString
(httpAddress));
}
if (policy.IsHttpsEnabled())
{
IPEndPoint httpsAddress = infoServer.GetConnectorAddress(connIdx);
conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressKey, NetUtils.GetHostPortString
(httpsAddress));
}
legacyOivImageDir = conf.Get(DFSConfigKeys.DfsNamenodeLegacyOivImageDirKey);
Log.Info("Checkpoint Period :" + checkpointConf.GetPeriod() + " secs " + "(" +
checkpointConf.GetPeriod() / 60 + " min)");
Log.Info("Log Size Trigger :" + checkpointConf.GetTxnCount() + " txns");
}
