/// <exception cref="System.Exception"/>
public virtual void TestDeleteBlockOnTransientStorage()
{
cluster = new MiniDFSCluster.Builder(Conf).StorageTypes(new StorageType[] { StorageType
.RamDisk, StorageType.Default }).NumDataNodes(1).Build();
try
{
cluster.WaitActive();
bpid = cluster.GetNamesystem().GetBlockPoolId();
DataNode dataNode = cluster.GetDataNodes()[0];
fds = DataNodeTestUtils.GetFSDataset(cluster.GetDataNodes()[0]);
client = cluster.GetFileSystem().GetClient();
scanner = new DirectoryScanner(dataNode, fds, Conf);
scanner.SetRetainDiffs(true);
FsDatasetTestUtil.StopLazyWriter(cluster.GetDataNodes()[0]);
// Create a file file on RAM_DISK
IList <LocatedBlock> blocks = CreateFile(GenericTestUtils.GetMethodName(), BlockLength
, true);
// Ensure no difference between volumeMap and disk.
Scan(1, 0, 0, 0, 0, 0);
// Make a copy of the block on DEFAULT storage and ensure that it is
// picked up by the scanner.
DuplicateBlock(blocks[0].GetBlock().GetBlockId());
Scan(2, 1, 0, 0, 0, 0, 1);
// Ensure that the copy on RAM_DISK was deleted.
VerifyStorageType(blocks[0].GetBlock().GetBlockId(), false);
Scan(1, 0, 0, 0, 0, 0);
}
finally
{
if (scanner != null)
{
scanner.Shutdown();
scanner = null;
}
cluster.Shutdown();
cluster = null;
}
}
/// <exception cref="System.Exception"/>
public virtual void RunTest(int parallelism)
{
cluster = new MiniDFSCluster.Builder(Conf).Build();
try
{
cluster.WaitActive();
bpid = cluster.GetNamesystem().GetBlockPoolId();
fds = DataNodeTestUtils.GetFSDataset(cluster.GetDataNodes()[0]);
client = cluster.GetFileSystem().GetClient();
Conf.SetInt(DFSConfigKeys.DfsDatanodeDirectoryscanThreadsKey, parallelism);
DataNode dataNode = cluster.GetDataNodes()[0];
scanner = new DirectoryScanner(dataNode, fds, Conf);
scanner.SetRetainDiffs(true);
// Add files with 100 blocks
CreateFile(GenericTestUtils.GetMethodName(), BlockLength * 100, false);
long totalBlocks = 100;
// Test1: No difference between volumeMap and disk
Scan(100, 0, 0, 0, 0, 0);
// Test2: block metafile is missing
long blockId = DeleteMetaFile();
Scan(totalBlocks, 1, 1, 0, 0, 1);
VerifyGenStamp(blockId, GenerationStamp.GrandfatherGenerationStamp);
Scan(totalBlocks, 0, 0, 0, 0, 0);
// Test3: block file is missing
blockId = DeleteBlockFile();
Scan(totalBlocks, 1, 0, 1, 0, 0);
totalBlocks--;
VerifyDeletion(blockId);
Scan(totalBlocks, 0, 0, 0, 0, 0);
// Test4: A block file exists for which there is no metafile and
// a block in memory
blockId = CreateBlockFile();
totalBlocks++;
Scan(totalBlocks, 1, 1, 0, 1, 0);
VerifyAddition(blockId, GenerationStamp.GrandfatherGenerationStamp, 0);
Scan(totalBlocks, 0, 0, 0, 0, 0);
// Test5: A metafile exists for which there is no block file and
// a block in memory
blockId = CreateMetaFile();
Scan(totalBlocks + 1, 1, 0, 1, 1, 0);
FilePath metafile = new FilePath(GetMetaFile(blockId));
NUnit.Framework.Assert.IsTrue(!metafile.Exists());
Scan(totalBlocks, 0, 0, 0, 0, 0);
// Test6: A block file and metafile exists for which there is no block in
// memory
blockId = CreateBlockMetaFile();
totalBlocks++;
Scan(totalBlocks, 1, 0, 0, 1, 0);
VerifyAddition(blockId, DefaultGenStamp, 0);
Scan(totalBlocks, 0, 0, 0, 0, 0);
// Test7: Delete bunch of metafiles
for (int i = 0; i < 10; i++)
{
blockId = DeleteMetaFile();
}
Scan(totalBlocks, 10, 10, 0, 0, 10);
Scan(totalBlocks, 0, 0, 0, 0, 0);
// Test8: Delete bunch of block files
for (int i_1 = 0; i_1 < 10; i_1++)
{
blockId = DeleteBlockFile();
}
Scan(totalBlocks, 10, 0, 10, 0, 0);
totalBlocks -= 10;
Scan(totalBlocks, 0, 0, 0, 0, 0);
// Test9: create a bunch of blocks files
for (int i_2 = 0; i_2 < 10; i_2++)
{
blockId = CreateBlockFile();
}
totalBlocks += 10;
Scan(totalBlocks, 10, 10, 0, 10, 0);
Scan(totalBlocks, 0, 0, 0, 0, 0);
// Test10: create a bunch of metafiles
for (int i_3 = 0; i_3 < 10; i_3++)
{
blockId = CreateMetaFile();
}
Scan(totalBlocks + 10, 10, 0, 10, 10, 0);
Scan(totalBlocks, 0, 0, 0, 0, 0);
// Test11: create a bunch block files and meta files
for (int i_4 = 0; i_4 < 10; i_4++)
{
blockId = CreateBlockMetaFile();
}
totalBlocks += 10;
Scan(totalBlocks, 10, 0, 0, 10, 0);
Scan(totalBlocks, 0, 0, 0, 0, 0);
// Test12: truncate block files to test block length mismatch
for (int i_5 = 0; i_5 < 10; i_5++)
{
TruncateBlockFile();
}
Scan(totalBlocks, 10, 0, 0, 0, 10);
Scan(totalBlocks, 0, 0, 0, 0, 0);
// Test13: all the conditions combined
CreateMetaFile();
CreateBlockFile();
CreateBlockMetaFile();
DeleteMetaFile();
DeleteBlockFile();
TruncateBlockFile();
Scan(totalBlocks + 3, 6, 2, 2, 3, 2);
Scan(totalBlocks + 1, 0, 0, 0, 0, 0);
// Test14: validate clean shutdown of DirectoryScanner
////assertTrue(scanner.getRunStatus()); //assumes "real" FSDataset, not sim
scanner.Shutdown();
NUnit.Framework.Assert.IsFalse(scanner.GetRunStatus());
}
finally
{
if (scanner != null)
{
scanner.Shutdown();
scanner = null;
}
cluster.Shutdown();
}
}
