public virtual void TestRefreshCallQueueProtocol()
{
RefreshCallQueueProtocolClientSideTranslatorPB translator = (RefreshCallQueueProtocolClientSideTranslatorPB
)NameNodeProxies.CreateNonHAProxy <RefreshCallQueueProtocol>(conf, nnAddress, UserGroupInformation
.GetCurrentUser(), true).GetProxy();
NUnit.Framework.Assert.IsTrue(translator.IsMethodSupported("refreshCallQueue"));
}
public virtual void TestGetUserMappingsProtocol()
{
GetUserMappingsProtocolClientSideTranslatorPB translator = (GetUserMappingsProtocolClientSideTranslatorPB
)NameNodeProxies.CreateNonHAProxy <GetUserMappingsProtocol>(conf, nnAddress, UserGroupInformation
.GetCurrentUser(), true).GetProxy();
NUnit.Framework.Assert.IsTrue(translator.IsMethodSupported("getGroupsForUser"));
}
public virtual void TestClientNamenodeProtocol()
{
ClientProtocol cp = NameNodeProxies.CreateNonHAProxy <ClientProtocol>(conf, nnAddress
, UserGroupInformation.GetCurrentUser(), true).GetProxy();
RpcClientUtil.IsMethodSupported(cp, typeof(ClientNamenodeProtocolPB), RPC.RpcKind
.RpcProtocolBuffer, RPC.GetProtocolVersion(typeof(ClientNamenodeProtocolPB)), "mkdirs"
);
}
public virtual void TesJournalProtocol()
{
JournalProtocolTranslatorPB translator = (JournalProtocolTranslatorPB)NameNodeProxies
.CreateNonHAProxy <JournalProtocol>(conf, nnAddress, UserGroupInformation.GetCurrentUser
(), true).GetProxy();
//Nameode doesn't implement JournalProtocol
NUnit.Framework.Assert.IsFalse(translator.IsMethodSupported("startLogSegment"));
}
public DummyLegacyFailoverProxyProvider(Configuration conf, URI uri, Type xface)
{
try
{
this.proxy = NameNodeProxies.CreateNonHAProxy(conf, NameNode.GetAddress(uri), xface
, UserGroupInformation.GetCurrentUser(), false).GetProxy();
this.xface = xface;
}
catch (IOException)
{
}
}
/// <summary>
/// Check whether logical URI is needed for the namenode and
/// the corresponding failover proxy provider in the config.
/// </summary>
/// <param name="conf">Configuration</param>
/// <param name="nameNodeUri">The URI of namenode</param>
/// <returns>true if logical URI is needed. false, if not needed.</returns>
/// <exception cref="System.IO.IOException">most likely due to misconfiguration.</exception>
public static bool UseLogicalUri(Configuration conf, URI nameNodeUri)
{
// Create the proxy provider. Actual proxy is not created.
AbstractNNFailoverProxyProvider <ClientProtocol> provider = NameNodeProxies.CreateFailoverProxyProvider
<ClientProtocol>(conf, nameNodeUri, false, null);
// No need to use logical URI since failover is not configured.
if (provider == null)
{
return(false);
}
// Check whether the failover proxy provider uses logical URI.
return(provider.UseLogicalURI());
}
public virtual void TestNamenodeProtocol()
{
NamenodeProtocol np = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nnAddress
, UserGroupInformation.GetCurrentUser(), true).GetProxy();
bool exists = RpcClientUtil.IsMethodSupported(np, typeof(NamenodeProtocolPB), RPC.RpcKind
.RpcProtocolBuffer, RPC.GetProtocolVersion(typeof(NamenodeProtocolPB)), "rollEditLog"
);
NUnit.Framework.Assert.IsTrue(exists);
exists = RpcClientUtil.IsMethodSupported(np, typeof(NamenodeProtocolPB), RPC.RpcKind
.RpcProtocolBuffer, RPC.GetProtocolVersion(typeof(NamenodeProtocolPB)), "bogusMethod"
);
NUnit.Framework.Assert.IsFalse(exists);
}
/// <summary>Get an RPC proxy for each NN in an HA nameservice.</summary>
/// <remarks>
/// Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC
/// call should be made on every NN in an HA nameservice, not just the active.
/// </remarks>
/// <param name="conf">configuration</param>
/// <param name="nsId">the nameservice to get all of the proxies for.</param>
/// <param name="xface">the protocol class.</param>
/// <returns>a list of RPC proxies for each NN in the nameservice.</returns>
/// <exception cref="System.IO.IOException">in the event of error.</exception>
public static IList <NameNodeProxies.ProxyAndInfo <T> > GetProxiesForAllNameNodesInNameservice
<T>(Configuration conf, string nsId)
{
System.Type xface = typeof(T);
IDictionary <string, IPEndPoint> nnAddresses = DFSUtil.GetRpcAddressesForNameserviceId
(conf, nsId, null);
IList <NameNodeProxies.ProxyAndInfo <T> > proxies = new AList <NameNodeProxies.ProxyAndInfo
<T> >(nnAddresses.Count);
foreach (IPEndPoint nnAddress in nnAddresses.Values)
{
NameNodeProxies.ProxyAndInfo <T> proxyInfo = null;
proxyInfo = NameNodeProxies.CreateNonHAProxy(conf, nnAddress, xface, UserGroupInformation
.GetCurrentUser(), false);
proxies.AddItem(proxyInfo);
}
return(proxies);
}
/* check if there are at least two nodes are on the same rack */
/// <exception cref="System.IO.IOException"/>
private void CheckFile(FileSystem fileSys, Path name, int repl)
{
Configuration conf = fileSys.GetConf();
ClientProtocol namenode = NameNodeProxies.CreateProxy <ClientProtocol>(conf, fileSys
.GetUri()).GetProxy();
WaitForBlockReplication(name.ToString(), namenode, Math.Min(numDatanodes, repl),
-1);
LocatedBlocks locations = namenode.GetBlockLocations(name.ToString(), 0, long.MaxValue
);
FileStatus stat = fileSys.GetFileStatus(name);
BlockLocation[] blockLocations = fileSys.GetFileBlockLocations(stat, 0L, long.MaxValue
);
// verify that rack locations match
NUnit.Framework.Assert.IsTrue(blockLocations.Length == locations.LocatedBlockCount
());
for (int i = 0; i < blockLocations.Length; i++)
{
LocatedBlock blk = locations.Get(i);
DatanodeInfo[] datanodes = blk.GetLocations();
string[] topologyPaths = blockLocations[i].GetTopologyPaths();
NUnit.Framework.Assert.IsTrue(topologyPaths.Length == datanodes.Length);
for (int j = 0; j < topologyPaths.Length; j++)
{
bool found = false;
for (int k = 0; k < racks.Length; k++)
{
if (topologyPaths[j].StartsWith(racks[k]))
{
found = true;
break;
}
}
NUnit.Framework.Assert.IsTrue(found);
}
}
bool isOnSameRack = true;
bool isNotOnSameRack = true;
foreach (LocatedBlock blk_1 in locations.GetLocatedBlocks())
{
DatanodeInfo[] datanodes = blk_1.GetLocations();
if (datanodes.Length <= 1)
{
break;
}
if (datanodes.Length == 2)
{
isNotOnSameRack = !(datanodes[0].GetNetworkLocation().Equals(datanodes[1].GetNetworkLocation
()));
break;
}
isOnSameRack = false;
isNotOnSameRack = false;
for (int i_1 = 0; i_1 < datanodes.Length - 1; i_1++)
{
Log.Info("datanode " + i_1 + ": " + datanodes[i_1]);
bool onRack = false;
for (int j = i_1 + 1; j < datanodes.Length; j++)
{
if (datanodes[i_1].GetNetworkLocation().Equals(datanodes[j].GetNetworkLocation()))
{
onRack = true;
}
}
if (onRack)
{
isOnSameRack = true;
}
if (!onRack)
{
isNotOnSameRack = true;
}
if (isOnSameRack && isNotOnSameRack)
{
break;
}
}
if (!isOnSameRack || !isNotOnSameRack)
{
break;
}
}
NUnit.Framework.Assert.IsTrue(isOnSameRack);
NUnit.Framework.Assert.IsTrue(isNotOnSameRack);
}
public virtual void TestGetBlocks()
{
Configuration Conf = new HdfsConfiguration();
short ReplicationFactor = (short)2;
int DefaultBlockSize = 1024;
Random r = new Random();
Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(ReplicationFactor
).Build();
try
{
cluster.WaitActive();
// create a file with two blocks
FileSystem fs = cluster.GetFileSystem();
FSDataOutputStream @out = fs.Create(new Path("/tmp.txt"), ReplicationFactor);
byte[] data = new byte[1024];
long fileLen = 2 * DefaultBlockSize;
long bytesToWrite = fileLen;
while (bytesToWrite > 0)
{
r.NextBytes(data);
int bytesToWriteNext = (1024 < bytesToWrite) ? 1024 : (int)bytesToWrite;
@out.Write(data, 0, bytesToWriteNext);
bytesToWrite -= bytesToWriteNext;
}
@out.Close();
// get blocks & data nodes
IList <LocatedBlock> locatedBlocks;
DatanodeInfo[] dataNodes = null;
bool notWritten;
do
{
DFSClient dfsclient = new DFSClient(NameNode.GetAddress(Conf), Conf);
locatedBlocks = dfsclient.GetNamenode().GetBlockLocations("/tmp.txt", 0, fileLen)
.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(2, locatedBlocks.Count);
notWritten = false;
for (int i = 0; i < 2; i++)
{
dataNodes = locatedBlocks[i].GetLocations();
if (dataNodes.Length != ReplicationFactor)
{
notWritten = true;
try
{
Sharpen.Thread.Sleep(10);
}
catch (Exception)
{
}
break;
}
}
}while (notWritten);
// get RPC client to namenode
IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort());
NamenodeProtocol namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(Conf, NameNode
.GetUri(addr)).GetProxy();
// get blocks of size fileLen from dataNodes[0]
BlocksWithLocations.BlockWithLocations[] locs;
locs = namenode.GetBlocks(dataNodes[0], fileLen).GetBlocks();
NUnit.Framework.Assert.AreEqual(locs.Length, 2);
NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
NUnit.Framework.Assert.AreEqual(locs[1].GetStorageIDs().Length, 2);
// get blocks of size BlockSize from dataNodes[0]
locs = namenode.GetBlocks(dataNodes[0], DefaultBlockSize).GetBlocks();
NUnit.Framework.Assert.AreEqual(locs.Length, 1);
NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
// get blocks of size 1 from dataNodes[0]
locs = namenode.GetBlocks(dataNodes[0], 1).GetBlocks();
NUnit.Framework.Assert.AreEqual(locs.Length, 1);
NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
// get blocks of size 0 from dataNodes[0]
GetBlocksWithException(namenode, dataNodes[0], 0);
// get blocks of size -1 from dataNodes[0]
GetBlocksWithException(namenode, dataNodes[0], -1);
// get blocks of size BlockSize from a non-existent datanode
DatanodeInfo info = DFSTestUtil.GetDatanodeInfo("1.2.3.4");
GetBlocksWithException(namenode, info, 2);
}
finally
{
cluster.Shutdown();
}
}