/// <summary>Disallow the scanner from scanning the given block pool.</summary>
/// <param name="bpid">The block pool id.</param>
public virtual void DisableBlockPoolId(string bpid)
{
lock (this)
{
IEnumerator <FsVolumeSpi.BlockIterator> i = blockIters.GetEnumerator();
while (i.HasNext())
{
FsVolumeSpi.BlockIterator iter = i.Next();
if (iter.GetBlockPoolId().Equals(bpid))
{
Log.Trace("{}: disabling scanning on block pool {}", this, bpid);
i.Remove();
IOUtils.Cleanup(null, iter);
if (curBlockIter == iter)
{
curBlockIter = null;
}
Sharpen.Runtime.Notify(this);
return;
}
}
Log.Warn("{}: can't remove block pool {}, because it was never " + "added.", this
, bpid);
}
}
/// <summary>
/// Find a usable block iterator.<p/>
/// We will consider available block iterators in order.
/// </summary>
/// <remarks>
/// Find a usable block iterator.<p/>
/// We will consider available block iterators in order. This property is
/// important so that we don't keep rescanning the same block pool id over
/// and over, while other block pools stay unscanned.<p/>
/// A block pool is always ready to scan if the iterator is not at EOF. If
/// the iterator is at EOF, the block pool will be ready to scan when
/// conf.scanPeriodMs milliseconds have elapsed since the iterator was last
/// rewound.<p/>
/// </remarks>
/// <returns>
/// 0 if we found a usable block iterator; the
/// length of time we should delay before
/// checking again otherwise.
/// </returns>
private long FindNextUsableBlockIter()
{
lock (this)
{
int numBlockIters = blockIters.Count;
if (numBlockIters == 0)
{
Log.Debug("{}: no block pools are registered.", this);
return(long.MaxValue);
}
int curIdx;
if (curBlockIter == null)
{
curIdx = 0;
}
else
{
curIdx = blockIters.IndexOf(curBlockIter);
Preconditions.CheckState(curIdx >= 0);
}
// Note that this has to be wall-clock time, not monotonic time. This is
// because the time saved in the cursor file is a wall-clock time. We do
// not want to save a monotonic time in the cursor file, because it resets
// every time the machine reboots (on most platforms).
long nowMs = Time.Now();
long minTimeoutMs = long.MaxValue;
for (int i = 0; i < numBlockIters; i++)
{
int idx = (curIdx + i + 1) % numBlockIters;
FsVolumeSpi.BlockIterator iter = blockIters[idx];
if (!iter.AtEnd())
{
Log.Info("Now scanning bpid {} on volume {}", iter.GetBlockPoolId(), volume.GetBasePath
());
curBlockIter = iter;
return(0L);
}
long iterStartMs = iter.GetIterStartMs();
long waitMs = (iterStartMs + conf.scanPeriodMs) - nowMs;
if (waitMs <= 0)
{
iter.Rewind();
Log.Info("Now rescanning bpid {} on volume {}, after more than " + "{} hour(s)",
iter.GetBlockPoolId(), volume.GetBasePath(), TimeUnit.Hours.Convert(conf.scanPeriodMs
, TimeUnit.Milliseconds));
curBlockIter = iter;
return(0L);
}
minTimeoutMs = Math.Min(minTimeoutMs, waitMs);
}
Log.Info("{}: no suitable block pools found to scan. Waiting {} ms.", this, minTimeoutMs
);
return(minTimeoutMs);
}
}
private void SaveBlockIterator(FsVolumeSpi.BlockIterator iter)
{
try
{
iter.Save();
}
catch (IOException e)
{
Log.Warn("{}: error saving {}.", this, iter, e);
}
}
// wake scanner thread.
/// <summary>Allow the scanner to scan the given block pool.</summary>
/// <param name="bpid">The block pool id.</param>
public virtual void EnableBlockPoolId(string bpid)
{
lock (this)
{
foreach (FsVolumeSpi.BlockIterator iter in blockIters)
{
if (iter.GetBlockPoolId().Equals(bpid))
{
Log.Warn("{}: already enabled scanning on block pool {}", this, bpid);
return;
}
}
FsVolumeSpi.BlockIterator iter_1 = null;
try
{
// Load a block iterator for the next block pool on the volume.
iter_1 = volume.LoadBlockIterator(bpid, BlockIteratorName);
Log.Trace("{}: loaded block iterator for {}.", this, bpid);
}
catch (FileNotFoundException e)
{
Log.Debug("{}: failed to load block iterator: " + e.Message, this);
}
catch (IOException e)
{
Log.Warn("{}: failed to load block iterator.", this, e);
}
if (iter_1 == null)
{
iter_1 = volume.NewBlockIterator(bpid, BlockIteratorName);
Log.Trace("{}: created new block iterator for {}.", this, bpid);
}
iter_1.SetMaxStalenessMs(conf.maxStalenessMs);
blockIters.AddItem(iter_1);
Sharpen.Runtime.Notify(this);
}
}
/// <summary>
/// Test iterating through a bunch of blocks in a volume using a volume
/// iterator.<p/>
/// We will rewind the iterator when about halfway through the blocks.
/// </summary>
/// <param name="numFiles">The number of files to create.</param>
/// <param name="maxStaleness">The maximum staleness to allow with the iterator.</param>
/// <exception cref="System.Exception"/>
private void TestVolumeIteratorImpl(int numFiles, long maxStaleness)
{
Configuration conf = new Configuration();
DisableBlockScanner(conf);
TestBlockScanner.TestContext ctx = new TestBlockScanner.TestContext(conf, 1);
ctx.CreateFiles(0, numFiles, 1);
NUnit.Framework.Assert.AreEqual(1, ctx.volumes.Count);
FsVolumeSpi volume = ctx.volumes[0];
ExtendedBlock savedBlock = null;
ExtendedBlock loadedBlock = null;
bool testedRewind = false;
bool testedSave = false;
bool testedLoad = false;
int blocksProcessed = 0;
int savedBlocksProcessed = 0;
try
{
BPOfferService[] bpos = ctx.datanode.GetAllBpOs();
NUnit.Framework.Assert.AreEqual(1, bpos.Length);
FsVolumeSpi.BlockIterator iter = volume.NewBlockIterator(ctx.bpids[0], "test");
NUnit.Framework.Assert.AreEqual(ctx.bpids[0], iter.GetBlockPoolId());
iter.SetMaxStalenessMs(maxStaleness);
while (true)
{
HashSet <ExtendedBlock> blocks = new HashSet <ExtendedBlock>();
for (int blockIdx = 0; blockIdx < numFiles; blockIdx++)
{
blocks.AddItem(ctx.GetFileBlock(0, blockIdx));
}
while (true)
{
ExtendedBlock block = iter.NextBlock();
if (block == null)
{
break;
}
blocksProcessed++;
Log.Info("BlockIterator for {} found block {}, blocksProcessed = {}", volume, block
, blocksProcessed);
if (testedSave && (savedBlock == null))
{
savedBlock = block;
}
if (testedLoad && (loadedBlock == null))
{
loadedBlock = block;
// The block that we get back right after loading the iterator
// should be the same block we got back right after saving
// the iterator.
NUnit.Framework.Assert.AreEqual(savedBlock, loadedBlock);
}
bool blockRemoved = blocks.Remove(block);
NUnit.Framework.Assert.IsTrue("Found unknown block " + block, blockRemoved);
if (blocksProcessed > (numFiles / 3))
{
if (!testedSave)
{
Log.Info("Processed {} blocks out of {}. Saving iterator.", blocksProcessed, numFiles
);
iter.Save();
testedSave = true;
savedBlocksProcessed = blocksProcessed;
}
}
if (blocksProcessed > (numFiles / 2))
{
if (!testedRewind)
{
Log.Info("Processed {} blocks out of {}. Rewinding iterator.", blocksProcessed,
numFiles);
iter.Rewind();
break;
}
}
if (blocksProcessed > ((2 * numFiles) / 3))
{
if (!testedLoad)
{
Log.Info("Processed {} blocks out of {}. Loading iterator.", blocksProcessed, numFiles
);
iter = volume.LoadBlockIterator(ctx.bpids[0], "test");
iter.SetMaxStalenessMs(maxStaleness);
break;
}
}
}
if (!testedRewind)
{
testedRewind = true;
blocksProcessed = 0;
Log.Info("Starting again at the beginning...");
continue;
}
if (!testedLoad)
{
testedLoad = true;
blocksProcessed = savedBlocksProcessed;
Log.Info("Starting again at the load point...");
continue;
}
NUnit.Framework.Assert.AreEqual(numFiles, blocksProcessed);
break;
}
}
finally
{
ctx.Close();
}
}
