/// <summary>
/// Count datanodes that have copies of the blocks for a file
/// put it into the map
/// </summary>
/// <param name="map"/>
/// <param name="path"/>
/// <param name="size"/>
/// <returns/>
/// <exception cref="System.IO.IOException"/>
private int CountNNBlocks(IDictionary <string, TestDataNodeVolumeFailure.BlockLocs
> map, string path, long size)
{
int total = 0;
NamenodeProtocols nn = cluster.GetNameNodeRpc();
IList <LocatedBlock> locatedBlocks = nn.GetBlockLocations(path, 0, size).GetLocatedBlocks
();
//System.out.println("Number of blocks: " + locatedBlocks.size());
foreach (LocatedBlock lb in locatedBlocks)
{
string blockId = string.Empty + lb.GetBlock().GetBlockId();
//System.out.print(blockId + ": ");
DatanodeInfo[] dn_locs = lb.GetLocations();
TestDataNodeVolumeFailure.BlockLocs bl = map[blockId];
if (bl == null)
{
bl = new TestDataNodeVolumeFailure.BlockLocs(this);
}
//System.out.print(dn_info.name+",");
total += dn_locs.Length;
bl.num_locs += dn_locs.Length;
map[blockId] = bl;
}
//System.out.println();
return(total);
}
/// <summary>
/// look for real blocks
/// by counting *.meta files in all the storage dirs
/// </summary>
/// <param name="map"/>
/// <returns/>
private int CountRealBlocks(IDictionary <string, TestDataNodeVolumeFailure.BlockLocs
> map)
{
int total = 0;
string bpid = cluster.GetNamesystem().GetBlockPoolId();
for (int i = 0; i < dn_num; i++)
{
for (int j = 0; j <= 1; j++)
{
FilePath storageDir = cluster.GetInstanceStorageDir(i, j);
FilePath dir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid);
if (dir == null)
{
System.Console.Out.WriteLine("dir is null for dn=" + i + " and data_dir=" + j);
continue;
}
IList <FilePath> res = MiniDFSCluster.GetAllBlockMetadataFiles(dir);
if (res == null)
{
System.Console.Out.WriteLine("res is null for dir = " + dir + " i=" + i + " and j="
+ j);
continue;
}
//System.out.println("for dn" + i + "." + j + ": " + dir + "=" + res.length+ " files");
//int ii = 0;
foreach (FilePath f in res)
{
string s = f.GetName();
// cut off "blk_-" at the beginning and ".meta" at the end
NUnit.Framework.Assert.IsNotNull("Block file name should not be null", s);
string bid = Sharpen.Runtime.Substring(s, s.IndexOf("_") + 1, s.LastIndexOf("_"));
//System.out.println(ii++ + ". block " + s + "; id=" + bid);
TestDataNodeVolumeFailure.BlockLocs val = map[bid];
if (val == null)
{
val = new TestDataNodeVolumeFailure.BlockLocs(this);
}
val.num_files++;
// one more file for the block
map[bid] = val;
}
//System.out.println("dir1="+dir.getPath() + "blocks=" + res.length);
//System.out.println("dir2="+dir2.getPath() + "blocks=" + res2.length);
total += res.Count;
}
}
return(total);
}
/// <summary>
/// verifies two things:
/// 1.
/// </summary>
/// <remarks>
/// verifies two things:
/// 1. number of locations of each block in the name node
/// matches number of actual files
/// 2. block files + pending block equals to total number of blocks that a file has
/// including the replication (HDFS file has 30 blocks, repl=2 - total 60
/// </remarks>
/// <param name="fn">- file name</param>
/// <param name="fs">- file size</param>
/// <exception cref="System.IO.IOException"/>
private void Verify(string fn, int fs)
{
// now count how many physical blocks are there
int totalReal = CountRealBlocks(block_map);
System.Console.Out.WriteLine("countRealBlocks counted " + totalReal + " blocks");
// count how many blocks store in NN structures.
int totalNN = CountNNBlocks(block_map, fn, fs);
System.Console.Out.WriteLine("countNNBlocks counted " + totalNN + " blocks");
foreach (string bid in block_map.Keys)
{
TestDataNodeVolumeFailure.BlockLocs bl = block_map[bid];
// System.out.println(bid + "->" + bl.num_files + "vs." + bl.num_locs);
// number of physical files (1 or 2) should be same as number of datanodes
// in the list of the block locations
NUnit.Framework.Assert.AreEqual("Num files should match num locations", bl.num_files
, bl.num_locs);
}
NUnit.Framework.Assert.AreEqual("Num physical blocks should match num stored in the NN"
, totalReal, totalNN);
// now check the number of under-replicated blocks
FSNamesystem fsn = cluster.GetNamesystem();
// force update of all the metric counts by calling computeDatanodeWork
BlockManagerTestUtil.GetComputedDatanodeWork(fsn.GetBlockManager());
// get all the counts
long underRepl = fsn.GetUnderReplicatedBlocks();
long pendRepl = fsn.GetPendingReplicationBlocks();
long totalRepl = underRepl + pendRepl;
System.Console.Out.WriteLine("underreplicated after = " + underRepl + " and pending repl ="
+ pendRepl + "; total underRepl = " + totalRepl);
System.Console.Out.WriteLine("total blocks (real and replicating):" + (totalReal
+ totalRepl) + " vs. all files blocks " + blocks_num * 2);
// together all the blocks should be equal to all real + all underreplicated
NUnit.Framework.Assert.AreEqual("Incorrect total block count", totalReal + totalRepl
, blocks_num * repl);
}
