public _Supplier_367(TestBlockScanner.TestContext ctx, ICollection <ExtendedBlock>
expectedBlocks, bool rescan, int NumExpectedBlocks)
{
this.ctx = ctx;
this.expectedBlocks = expectedBlocks;
this.rescan = rescan;
this.NumExpectedBlocks = NumExpectedBlocks;
}
/// <exception cref="System.Exception"/>
private void TestScanAllBlocksImpl(bool rescan)
{
Configuration conf = new Configuration();
conf.SetLong(DFSConfigKeys.DfsBlockScannerVolumeBytesPerSecond, 1048576L);
if (rescan)
{
conf.SetLong(BlockScanner.Conf.InternalDfsDatanodeScanPeriodMs, 100L);
}
else
{
conf.SetLong(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, 100L);
}
conf.Set(BlockScanner.Conf.InternalVolumeScannerScanResultHandler, typeof(TestBlockScanner.TestScanResultHandler
).FullName);
TestBlockScanner.TestContext ctx = new TestBlockScanner.TestContext(conf, 1);
int NumExpectedBlocks = 10;
ctx.CreateFiles(0, NumExpectedBlocks, 1);
ICollection <ExtendedBlock> expectedBlocks = new HashSet <ExtendedBlock>();
for (int i = 0; i < NumExpectedBlocks; i++)
{
expectedBlocks.AddItem(ctx.GetFileBlock(0, i));
}
TestBlockScanner.TestScanResultHandler.Info info = TestBlockScanner.TestScanResultHandler
.GetInfo(ctx.volumes[0]);
lock (info)
{
info.shouldRun = true;
Sharpen.Runtime.Notify(info);
}
GenericTestUtils.WaitFor(new _Supplier_367(ctx, expectedBlocks, rescan, NumExpectedBlocks
), 10, 60000);
if (!rescan)
{
lock (info)
{
NUnit.Framework.Assert.AreEqual(NumExpectedBlocks, info.blocksScanned);
}
VolumeScanner.Statistics stats = ctx.blockScanner.GetVolumeStats(ctx.volumes[0].GetStorageID
());
NUnit.Framework.Assert.AreEqual(5 * NumExpectedBlocks, stats.bytesScannedInPastHour
);
NUnit.Framework.Assert.AreEqual(NumExpectedBlocks, stats.blocksScannedSinceRestart
);
NUnit.Framework.Assert.AreEqual(NumExpectedBlocks, stats.blocksScannedInCurrentPeriod
);
NUnit.Framework.Assert.AreEqual(0, stats.scanErrorsSinceRestart);
NUnit.Framework.Assert.AreEqual(1, stats.scansSinceRestart);
}
ctx.Close();
}
/// <exception cref="System.Exception"/>
public virtual void TestDisableVolumeScanner()
{
Configuration conf = new Configuration();
DisableBlockScanner(conf);
TestBlockScanner.TestContext ctx = new TestBlockScanner.TestContext(conf, 1);
try
{
NUnit.Framework.Assert.IsFalse(ctx.datanode.GetBlockScanner().IsEnabled());
}
finally
{
ctx.Close();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestMultipleBlockPoolScanning()
{
Configuration conf = new Configuration();
conf.SetLong(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, 100L);
conf.Set(BlockScanner.Conf.InternalVolumeScannerScanResultHandler, typeof(TestBlockScanner.TestScanResultHandler
).FullName);
TestBlockScanner.TestContext ctx = new TestBlockScanner.TestContext(conf, 3);
// We scan 5 bytes per file (1 byte in file, 4 bytes of checksum)
int BytesScannedPerFile = 5;
int[] NumFiles = new int[] { 1, 5, 10 };
int TotalFiles = 0;
for (int i = 0; i < NumFiles.Length; i++)
{
TotalFiles += NumFiles[i];
}
ctx.CreateFiles(0, NumFiles[0], 1);
ctx.CreateFiles(0, NumFiles[1], 1);
ctx.CreateFiles(0, NumFiles[2], 1);
// start scanning
TestBlockScanner.TestScanResultHandler.Info info = TestBlockScanner.TestScanResultHandler
.GetInfo(ctx.volumes[0]);
lock (info)
{
info.shouldRun = true;
Sharpen.Runtime.Notify(info);
}
// Wait for all the block pools to be scanned.
GenericTestUtils.WaitFor(new _Supplier_632(info, ctx), 3, 30000);
VolumeScanner.Statistics stats = ctx.blockScanner.GetVolumeStats(ctx.volumes[0].GetStorageID
());
NUnit.Framework.Assert.AreEqual(TotalFiles, stats.blocksScannedSinceRestart);
NUnit.Framework.Assert.AreEqual(BytesScannedPerFile * TotalFiles, stats.bytesScannedInPastHour
);
ctx.Close();
}
/// <exception cref="System.Exception"/>
public virtual void TestCorruptBlockHandling()
{
Configuration conf = new Configuration();
conf.SetLong(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, 100L);
conf.Set(BlockScanner.Conf.InternalVolumeScannerScanResultHandler, typeof(TestBlockScanner.TestScanResultHandler
).FullName);
TestBlockScanner.TestContext ctx = new TestBlockScanner.TestContext(conf, 1);
int NumExpectedBlocks = 5;
int CorruptIndex = 3;
ctx.CreateFiles(0, NumExpectedBlocks, 4);
ExtendedBlock badBlock = ctx.GetFileBlock(0, CorruptIndex);
ctx.cluster.CorruptBlockOnDataNodes(badBlock);
TestBlockScanner.TestScanResultHandler.Info info = TestBlockScanner.TestScanResultHandler
.GetInfo(ctx.volumes[0]);
lock (info)
{
info.shouldRun = true;
Sharpen.Runtime.Notify(info);
}
GenericTestUtils.WaitFor(new _Supplier_488(info, NumExpectedBlocks), 3, 30000);
lock (info)
{
NUnit.Framework.Assert.IsTrue(info.badBlocks.Contains(badBlock));
for (int i = 0; i < NumExpectedBlocks; i++)
{
if (i != CorruptIndex)
{
ExtendedBlock block = ctx.GetFileBlock(0, i);
NUnit.Framework.Assert.IsTrue(info.goodBlocks.Contains(block));
}
}
}
ctx.Close();
}
/// <summary>Test that we don't scan too many blocks per second.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestScanRateLimit()
{
Configuration conf = new Configuration();
// Limit scan bytes per second dramatically
conf.SetLong(DFSConfigKeys.DfsBlockScannerVolumeBytesPerSecond, 4096L);
// Scan continuously
conf.SetLong(BlockScanner.Conf.InternalDfsDatanodeScanPeriodMs, 1L);
conf.Set(BlockScanner.Conf.InternalVolumeScannerScanResultHandler, typeof(TestBlockScanner.TestScanResultHandler
).FullName);
TestBlockScanner.TestContext ctx = new TestBlockScanner.TestContext(conf, 1);
int NumExpectedBlocks = 5;
ctx.CreateFiles(0, NumExpectedBlocks, 4096);
TestBlockScanner.TestScanResultHandler.Info info = TestBlockScanner.TestScanResultHandler
.GetInfo(ctx.volumes[0]);
long startMs = Time.MonotonicNow();
lock (info)
{
info.shouldRun = true;
Sharpen.Runtime.Notify(info);
}
GenericTestUtils.WaitFor(new _Supplier_448(info), 1, 30000);
Sharpen.Thread.Sleep(2000);
lock (info)
{
long endMs = Time.MonotonicNow();
// Should scan no more than one block a second.
long seconds = ((endMs + 999 - startMs) / 1000);
long maxBlocksScanned = seconds * 1;
NUnit.Framework.Assert.IsTrue("The number of blocks scanned is too large. Scanned "
+ info.blocksScanned + " blocks; only expected to scan at most " + maxBlocksScanned
+ " in " + seconds + " seconds.", info.blocksScanned <= maxBlocksScanned);
}
ctx.Close();
}
/// <summary>
/// Test that we can mark certain blocks as suspect, and get them quickly
/// rescanned that way.
/// </summary>
/// <remarks>
/// Test that we can mark certain blocks as suspect, and get them quickly
/// rescanned that way. See HDFS-7686 and HDFS-7548.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestMarkSuspectBlock()
{
Configuration conf = new Configuration();
// Set a really long scan period.
conf.SetLong(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, 100L);
conf.Set(BlockScanner.Conf.InternalVolumeScannerScanResultHandler, typeof(TestBlockScanner.TestScanResultHandler
).FullName);
conf.SetLong(BlockScanner.Conf.InternalDfsBlockScannerCursorSaveIntervalMs, 0L);
TestBlockScanner.TestContext ctx = new TestBlockScanner.TestContext(conf, 1);
int NumExpectedBlocks = 10;
ctx.CreateFiles(0, NumExpectedBlocks, 1);
TestBlockScanner.TestScanResultHandler.Info info = TestBlockScanner.TestScanResultHandler
.GetInfo(ctx.volumes[0]);
string storageID = ctx.datanode.GetFSDataset().GetVolumes()[0].GetStorageID();
lock (info)
{
info.sem = Sharpen.Extensions.CreateSemaphore(4);
info.shouldRun = true;
Sharpen.Runtime.Notify(info);
}
// Scan the first 4 blocks
Log.Info("Waiting for the first 4 blocks to be scanned.");
GenericTestUtils.WaitFor(new _Supplier_725(info), 50, 30000);
// We should have scanned 4 blocks
lock (info)
{
NUnit.Framework.Assert.AreEqual("Expected 4 good blocks.", 4, info.goodBlocks.Count
);
info.goodBlocks.Clear();
NUnit.Framework.Assert.AreEqual("Expected 4 blocksScanned", 4, info.blocksScanned
);
NUnit.Framework.Assert.AreEqual("Did not expect bad blocks.", 0, info.badBlocks.Count
);
info.blocksScanned = 0;
}
ExtendedBlock first = ctx.GetFileBlock(0, 0);
ctx.datanode.GetBlockScanner().MarkSuspectBlock(storageID, first);
// When we increment the semaphore, the TestScanResultHandler will finish
// adding the block that it was scanning previously (the 5th block).
// We increment the semaphore twice so that the handler will also
// get a chance to see the suspect block which we just requested the
// VolumeScanner to process.
info.sem.Release(2);
Log.Info("Waiting for 2 more blocks to be scanned.");
GenericTestUtils.WaitFor(new _Supplier_758(info), 50, 30000);
lock (info)
{
NUnit.Framework.Assert.IsTrue("Expected block " + first + " to have been scanned."
, info.goodBlocks.Contains(first));
NUnit.Framework.Assert.AreEqual(2, info.goodBlocks.Count);
info.goodBlocks.Clear();
NUnit.Framework.Assert.AreEqual("Did not expect bad blocks.", 0, info.badBlocks.Count
);
NUnit.Framework.Assert.AreEqual(2, info.blocksScanned);
info.blocksScanned = 0;
}
// Re-mark the same block as suspect.
ctx.datanode.GetBlockScanner().MarkSuspectBlock(storageID, first);
info.sem.Release(10);
Log.Info("Waiting for 5 more blocks to be scanned.");
GenericTestUtils.WaitFor(new _Supplier_788(info), 50, 30000);
lock (info)
{
NUnit.Framework.Assert.AreEqual(5, info.goodBlocks.Count);
NUnit.Framework.Assert.AreEqual(0, info.badBlocks.Count);
NUnit.Framework.Assert.AreEqual(5, info.blocksScanned);
// We should not have rescanned the "suspect block",
// because it was recently rescanned by the suspect block system.
// This is a test of the "suspect block" rate limiting.
NUnit.Framework.Assert.IsFalse("We should not " + "have rescanned block " + first
+ ", because it should have been " + "in recentSuspectBlocks.", info.goodBlocks
.Contains(first));
info.blocksScanned = 0;
}
}
public _Supplier_632(TestBlockScanner.TestScanResultHandler.Info info, TestBlockScanner.TestContext
ctx)
{
this.info = info;
this.ctx = ctx;
}
/// <summary>
/// Test that we save the scan cursor when shutting down the datanode, and
/// restart scanning from there when the datanode is restarted.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestDatanodeCursor()
{
Configuration conf = new Configuration();
conf.SetLong(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, 100L);
conf.Set(BlockScanner.Conf.InternalVolumeScannerScanResultHandler, typeof(TestBlockScanner.TestScanResultHandler
).FullName);
conf.SetLong(BlockScanner.Conf.InternalDfsBlockScannerCursorSaveIntervalMs, 0L);
TestBlockScanner.TestContext ctx = new TestBlockScanner.TestContext(conf, 1);
int NumExpectedBlocks = 10;
ctx.CreateFiles(0, NumExpectedBlocks, 1);
TestBlockScanner.TestScanResultHandler.Info info = TestBlockScanner.TestScanResultHandler
.GetInfo(ctx.volumes[0]);
lock (info)
{
info.sem = Sharpen.Extensions.CreateSemaphore(5);
info.shouldRun = true;
Sharpen.Runtime.Notify(info);
}
// Scan the first 5 blocks
GenericTestUtils.WaitFor(new _Supplier_530(info), 3, 30000);
lock (info)
{
NUnit.Framework.Assert.AreEqual(5, info.goodBlocks.Count);
NUnit.Framework.Assert.AreEqual(5, info.blocksScanned);
info.shouldRun = false;
}
ctx.datanode.Shutdown();
string vPath = ctx.volumes[0].GetBasePath();
FilePath cursorPath = new FilePath(new FilePath(new FilePath(vPath, "current"), ctx
.bpids[0]), "scanner.cursor");
NUnit.Framework.Assert.IsTrue("Failed to find cursor save file in " + cursorPath.
GetAbsolutePath(), cursorPath.Exists());
ICollection <ExtendedBlock> prevGoodBlocks = new HashSet <ExtendedBlock>();
lock (info)
{
info.sem = Sharpen.Extensions.CreateSemaphore(4);
Sharpen.Collections.AddAll(prevGoodBlocks, info.goodBlocks);
info.goodBlocks.Clear();
}
// The block that we were scanning when we shut down the DN won't get
// recorded.
// After restarting the datanode, we should scan the next 4 blocks.
ctx.cluster.RestartDataNode(0);
lock (info)
{
info.shouldRun = true;
Sharpen.Runtime.Notify(info);
}
GenericTestUtils.WaitFor(new _Supplier_564(info), 3, 30000);
lock (info)
{
NUnit.Framework.Assert.AreEqual(4, info.goodBlocks.Count);
Sharpen.Collections.AddAll(info.goodBlocks, prevGoodBlocks);
NUnit.Framework.Assert.AreEqual(9, info.goodBlocks.Count);
NUnit.Framework.Assert.AreEqual(9, info.blocksScanned);
}
ctx.datanode.Shutdown();
// After restarting the datanode, we should not scan any more blocks.
// This is because we reached the end of the block pool earlier, and
// the scan period is much, much longer than the test time.
lock (info)
{
info.sem = null;
info.shouldRun = false;
info.goodBlocks.Clear();
}
ctx.cluster.RestartDataNode(0);
lock (info)
{
info.shouldRun = true;
Sharpen.Runtime.Notify(info);
}
Sharpen.Thread.Sleep(3000);
lock (info)
{
NUnit.Framework.Assert.IsTrue(info.goodBlocks.IsEmpty());
}
ctx.Close();
}
/// <summary>
/// Test iterating through a bunch of blocks in a volume using a volume
/// iterator.<p/>
/// We will rewind the iterator when about halfway through the blocks.
/// </summary>
/// <param name="numFiles">The number of files to create.</param>
/// <param name="maxStaleness">The maximum staleness to allow with the iterator.</param>
/// <exception cref="System.Exception"/>
private void TestVolumeIteratorImpl(int numFiles, long maxStaleness)
{
Configuration conf = new Configuration();
DisableBlockScanner(conf);
TestBlockScanner.TestContext ctx = new TestBlockScanner.TestContext(conf, 1);
ctx.CreateFiles(0, numFiles, 1);
NUnit.Framework.Assert.AreEqual(1, ctx.volumes.Count);
FsVolumeSpi volume = ctx.volumes[0];
ExtendedBlock savedBlock = null;
ExtendedBlock loadedBlock = null;
bool testedRewind = false;
bool testedSave = false;
bool testedLoad = false;
int blocksProcessed = 0;
int savedBlocksProcessed = 0;
try
{
BPOfferService[] bpos = ctx.datanode.GetAllBpOs();
NUnit.Framework.Assert.AreEqual(1, bpos.Length);
FsVolumeSpi.BlockIterator iter = volume.NewBlockIterator(ctx.bpids[0], "test");
NUnit.Framework.Assert.AreEqual(ctx.bpids[0], iter.GetBlockPoolId());
iter.SetMaxStalenessMs(maxStaleness);
while (true)
{
HashSet <ExtendedBlock> blocks = new HashSet <ExtendedBlock>();
for (int blockIdx = 0; blockIdx < numFiles; blockIdx++)
{
blocks.AddItem(ctx.GetFileBlock(0, blockIdx));
}
while (true)
{
ExtendedBlock block = iter.NextBlock();
if (block == null)
{
break;
}
blocksProcessed++;
Log.Info("BlockIterator for {} found block {}, blocksProcessed = {}", volume, block
, blocksProcessed);
if (testedSave && (savedBlock == null))
{
savedBlock = block;
}
if (testedLoad && (loadedBlock == null))
{
loadedBlock = block;
// The block that we get back right after loading the iterator
// should be the same block we got back right after saving
// the iterator.
NUnit.Framework.Assert.AreEqual(savedBlock, loadedBlock);
}
bool blockRemoved = blocks.Remove(block);
NUnit.Framework.Assert.IsTrue("Found unknown block " + block, blockRemoved);
if (blocksProcessed > (numFiles / 3))
{
if (!testedSave)
{
Log.Info("Processed {} blocks out of {}. Saving iterator.", blocksProcessed, numFiles
);
iter.Save();
testedSave = true;
savedBlocksProcessed = blocksProcessed;
}
}
if (blocksProcessed > (numFiles / 2))
{
if (!testedRewind)
{
Log.Info("Processed {} blocks out of {}. Rewinding iterator.", blocksProcessed,
numFiles);
iter.Rewind();
break;
}
}
if (blocksProcessed > ((2 * numFiles) / 3))
{
if (!testedLoad)
{
Log.Info("Processed {} blocks out of {}. Loading iterator.", blocksProcessed, numFiles
);
iter = volume.LoadBlockIterator(ctx.bpids[0], "test");
iter.SetMaxStalenessMs(maxStaleness);
break;
}
}
}
if (!testedRewind)
{
testedRewind = true;
blocksProcessed = 0;
Log.Info("Starting again at the beginning...");
continue;
}
if (!testedLoad)
{
testedLoad = true;
blocksProcessed = savedBlocksProcessed;
Log.Info("Starting again at the load point...");
continue;
}
NUnit.Framework.Assert.AreEqual(numFiles, blocksProcessed);
break;
}
}
finally
{
ctx.Close();
}
}
