/// <exception cref="System.IO.IOException"/>
private void TestDataNodeRedirect(Path path)
{
// Create the file
if (hdfs.Exists(path))
{
hdfs.Delete(path, true);
}
FSDataOutputStream @out = hdfs.Create(path, (short)1);
@out.WriteBytes("0123456789");
@out.Close();
// Get the path's block location so we can determine
// if we were redirected to the right DN.
BlockLocation[] locations = hdfs.GetFileBlockLocations(path, 0, 10);
string xferAddr = locations[0].GetNames()[0];
// Connect to the NN to get redirected
Uri u = hftpFs.GetNamenodeURL("/data" + ServletUtil.EncodePath(path.ToUri().GetPath
()), "ugi=userx,groupy");
HttpURLConnection conn = (HttpURLConnection)u.OpenConnection();
HttpURLConnection.SetFollowRedirects(true);
conn.Connect();
conn.GetInputStream();
bool @checked = false;
// Find the datanode that has the block according to locations
// and check that the URL was redirected to this DN's info port
foreach (DataNode node in cluster.GetDataNodes())
{
DatanodeRegistration dnR = DataNodeTestUtils.GetDNRegistrationForBP(node, blockPoolId
);
if (dnR.GetXferAddr().Equals(xferAddr))
{
@checked = true;
NUnit.Framework.Assert.AreEqual(dnR.GetInfoPort(), conn.GetURL().Port);
}
}
NUnit.Framework.Assert.IsTrue("The test never checked that location of " + "the block and hftp desitnation are the same"
, @checked);
}
public virtual void TestNumVersionsReportedCorrect()
{
//Create the DatanodeManager which will be tested
FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>();
Org.Mockito.Mockito.When(fsn.HasWriteLock()).ThenReturn(true);
DatanodeManager dm = new DatanodeManager(Org.Mockito.Mockito.Mock <BlockManager>()
, fsn, new Configuration());
//Seed the RNG with a known value so test failures are easier to reproduce
Random rng = new Random();
int seed = rng.Next();
rng = new Random(seed);
Log.Info("Using seed " + seed + " for testing");
//A map of the Storage IDs to the DN registration it was registered with
Dictionary <string, DatanodeRegistration> sIdToDnReg = new Dictionary <string, DatanodeRegistration
>();
for (int i = 0; i < NumIterations; ++i)
{
//If true, remove a node for every 3rd time (if there's one)
if (rng.NextBoolean() && i % 3 == 0 && sIdToDnReg.Count != 0)
{
//Pick a random node.
int randomIndex = rng.Next() % sIdToDnReg.Count;
//Iterate to that random position
IEnumerator <KeyValuePair <string, DatanodeRegistration> > it = sIdToDnReg.GetEnumerator
();
for (int j = 0; j < randomIndex - 1; ++j)
{
it.Next();
}
DatanodeRegistration toRemove = it.Next().Value;
Log.Info("Removing node " + toRemove.GetDatanodeUuid() + " ip " + toRemove.GetXferAddr
() + " version : " + toRemove.GetSoftwareVersion());
//Remove that random node
dm.RemoveDatanode(toRemove);
it.Remove();
}
else
{
// Otherwise register a node. This node may be a new / an old one
//Pick a random storageID to register.
string storageID = "someStorageID" + rng.Next(5000);
DatanodeRegistration dr = Org.Mockito.Mockito.Mock <DatanodeRegistration>();
Org.Mockito.Mockito.When(dr.GetDatanodeUuid()).ThenReturn(storageID);
//If this storageID had already been registered before
if (sIdToDnReg.Contains(storageID))
{
dr = sIdToDnReg[storageID];
//Half of the times, change the IP address
if (rng.NextBoolean())
{
dr.SetIpAddr(dr.GetIpAddr() + "newIP");
}
}
else
{
//This storageID has never been registered
//Ensure IP address is unique to storageID
string ip = "someIP" + storageID;
Org.Mockito.Mockito.When(dr.GetIpAddr()).ThenReturn(ip);
Org.Mockito.Mockito.When(dr.GetXferAddr()).ThenReturn(ip + ":9000");
Org.Mockito.Mockito.When(dr.GetXferPort()).ThenReturn(9000);
}
//Pick a random version to register with
Org.Mockito.Mockito.When(dr.GetSoftwareVersion()).ThenReturn("version" + rng.Next
(5));
Log.Info("Registering node storageID: " + dr.GetDatanodeUuid() + ", version: " +
dr.GetSoftwareVersion() + ", IP address: " + dr.GetXferAddr());
//Register this random node
dm.RegisterDatanode(dr);
sIdToDnReg[storageID] = dr;
}
//Verify DatanodeManager still has the right count
IDictionary <string, int> mapToCheck = dm.GetDatanodesSoftwareVersions();
//Remove counts from versions and make sure that after removing all nodes
//mapToCheck is empty
foreach (KeyValuePair <string, DatanodeRegistration> it_1 in sIdToDnReg)
{
string ver = it_1.Value.GetSoftwareVersion();
if (!mapToCheck.Contains(ver))
{
throw new Exception("The correct number of datanodes of a " + "version was not found on iteration "
+ i);
}
mapToCheck[ver] = mapToCheck[ver] - 1;
if (mapToCheck[ver] == 0)
{
Sharpen.Collections.Remove(mapToCheck, ver);
}
}
foreach (KeyValuePair <string, int> entry in mapToCheck)
{
Log.Info("Still in map: " + entry.Key + " has " + entry.Value);
}
NUnit.Framework.Assert.AreEqual("The map of version counts returned by DatanodeManager was"
+ " not what it was expected to be on iteration " + i, 0, mapToCheck.Count);
}
}
public virtual void TestSortLocatedBlocks()
{
// create the DatanodeManager which will be tested
FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>();
Org.Mockito.Mockito.When(fsn.HasWriteLock()).ThenReturn(true);
DatanodeManager dm = new DatanodeManager(Org.Mockito.Mockito.Mock <BlockManager>()
, fsn, new Configuration());
// register 5 datanodes, each with different storage ID and type
DatanodeInfo[] locs = new DatanodeInfo[5];
string[] storageIDs = new string[5];
StorageType[] storageTypes = new StorageType[] { StorageType.Archive, StorageType
.Default, StorageType.Disk, StorageType.RamDisk, StorageType.Ssd };
for (int i = 0; i < 5; i++)
{
// register new datanode
string uuid = "UUID-" + i;
string ip = "IP-" + i;
DatanodeRegistration dr = Org.Mockito.Mockito.Mock <DatanodeRegistration>();
Org.Mockito.Mockito.When(dr.GetDatanodeUuid()).ThenReturn(uuid);
Org.Mockito.Mockito.When(dr.GetIpAddr()).ThenReturn(ip);
Org.Mockito.Mockito.When(dr.GetXferAddr()).ThenReturn(ip + ":9000");
Org.Mockito.Mockito.When(dr.GetXferPort()).ThenReturn(9000);
Org.Mockito.Mockito.When(dr.GetSoftwareVersion()).ThenReturn("version1");
dm.RegisterDatanode(dr);
// get location and storage information
locs[i] = dm.GetDatanode(uuid);
storageIDs[i] = "storageID-" + i;
}
// set first 2 locations as decomissioned
locs[0].SetDecommissioned();
locs[1].SetDecommissioned();
// create LocatedBlock with above locations
ExtendedBlock b = new ExtendedBlock("somePoolID", 1234);
LocatedBlock block = new LocatedBlock(b, locs, storageIDs, storageTypes);
IList <LocatedBlock> blocks = new AList <LocatedBlock>();
blocks.AddItem(block);
string targetIp = locs[4].GetIpAddr();
// sort block locations
dm.SortLocatedBlocks(targetIp, blocks);
// check that storage IDs/types are aligned with datanode locs
DatanodeInfo[] sortedLocs = block.GetLocations();
storageIDs = block.GetStorageIDs();
storageTypes = block.GetStorageTypes();
Assert.AssertThat(sortedLocs.Length, IS.Is(5));
Assert.AssertThat(storageIDs.Length, IS.Is(5));
Assert.AssertThat(storageTypes.Length, IS.Is(5));
for (int i_1 = 0; i_1 < sortedLocs.Length; i_1++)
{
Assert.AssertThat(((DatanodeInfoWithStorage)sortedLocs[i_1]).GetStorageID(), IS.Is
(storageIDs[i_1]));
Assert.AssertThat(((DatanodeInfoWithStorage)sortedLocs[i_1]).GetStorageType(), IS.Is
(storageTypes[i_1]));
}
// Ensure the local node is first.
Assert.AssertThat(sortedLocs[0].GetIpAddr(), IS.Is(targetIp));
// Ensure the two decommissioned DNs were moved to the end.
Assert.AssertThat(sortedLocs[sortedLocs.Length - 1].GetAdminState(), IS.Is(DatanodeInfo.AdminStates
.Decommissioned));
Assert.AssertThat(sortedLocs[sortedLocs.Length - 2].GetAdminState(), IS.Is(DatanodeInfo.AdminStates
.Decommissioned));
}