/// <summary>Constructor</summary>
/// <param name="blockId">block id</param>
/// <param name="len">replica length</param>
/// <param name="genStamp">replica generation stamp</param>
/// <param name="vol">volume where replica is located</param>
/// <param name="dir">directory path where block and meta files are located</param>
internal ReplicaInfo(long blockId, long len, long genStamp, FsVolumeSpi vol, FilePath
dir)
: base(blockId, len, genStamp)
{
this.volume = vol;
SetDirInternal(dir);
}
/// <summary>Block is not found on the disk</summary>
private void AddDifference(List <DirectoryScanner.ScanInfo> diffRecord, DirectoryScanner.Stats
statsRecord, long blockId, FsVolumeSpi vol)
{
statsRecord.missingBlockFile++;
statsRecord.missingMetaFile++;
diffRecord.AddItem(new DirectoryScanner.ScanInfo(blockId, null, null, vol));
}
internal ScanInfo(long blockId, FilePath blockFile, FilePath metaFile, FsVolumeSpi
vol)
{
this.blockId = blockId;
string condensedVolPath = vol == null ? null : GetCondensedPath(vol.GetBasePath()
);
this.blockSuffix = blockFile == null ? null : GetSuffix(blockFile, condensedVolPath
);
this.blockFileLength = (blockFile != null) ? blockFile.Length() : 0;
if (metaFile == null)
{
this.metaSuffix = null;
}
else
{
if (blockFile == null)
{
this.metaSuffix = GetSuffix(metaFile, condensedVolPath);
}
else
{
this.metaSuffix = GetSuffix(metaFile, condensedVolPath + blockSuffix);
}
}
this.volume = vol;
}
/// <summary>Constructor</summary>
/// <param name="blockId">block id</param>
/// <param name="len">replica length</param>
/// <param name="genStamp">replica generation stamp</param>
/// <param name="vol">volume where replica is located</param>
/// <param name="dir">directory path where block and meta files are located</param>
/// <param name="writer">a thread that is writing to this replica</param>
/// <param name="bytesToReserve">
/// disk space to reserve for this replica, based on
/// the estimated maximum block length.
/// </param>
internal ReplicaInPipeline(long blockId, long len, long genStamp, FsVolumeSpi vol
, FilePath dir, Sharpen.Thread writer, long bytesToReserve)
: base(blockId, len, genStamp, vol, dir)
{
this.bytesAcked = len;
this.bytesOnDisk = len;
this.writer = writer;
this.bytesReserved = bytesToReserve;
}
internal static TestBlockScanner.TestScanResultHandler.Info GetInfo(FsVolumeSpi volume
)
{
TestBlockScanner.TestScanResultHandler.Info newInfo = new TestBlockScanner.TestScanResultHandler.Info
();
TestBlockScanner.TestScanResultHandler.Info prevInfo = infos.PutIfAbsent(volume.GetStorageID
(), newInfo);
return(prevInfo == null ? newInfo : prevInfo);
}
internal RamDiskReplica(string bpid, long blockId, FsVolumeImpl ramDiskVolume)
{
this.bpid = bpid;
this.blockId = blockId;
this.ramDiskVolume = ramDiskVolume;
lazyPersistVolume = null;
savedMetaFile = null;
savedBlockFile = null;
creationTime = Time.MonotonicNow();
isPersisted = false;
}
/// <summary>Is the given volume still valid in the dataset?</summary>
private static bool IsValid <_T0>(FsDatasetSpi <_T0> dataset, FsVolumeSpi volume)
where _T0 : FsVolumeSpi
{
foreach (FsVolumeSpi vol in dataset.GetVolumes())
{
if (vol == volume)
{
return(true);
}
}
return(false);
}
public virtual void TestBlockHasMultipleReplicasOnSameDN()
{
string filename = MakeFileName(GenericTestUtils.GetMethodName());
Path filePath = new Path(filename);
// Write out a file with a few blocks.
DFSTestUtil.CreateFile(fs, filePath, BlockSize, BlockSize * NumBlocks, BlockSize,
NumDatanodes, seed);
// Get the block list for the file with the block locations.
LocatedBlocks locatedBlocks = client.GetLocatedBlocks(filePath.ToString(), 0, BlockSize
* NumBlocks);
// Generate a fake block report from one of the DataNodes, such
// that it reports one copy of each block on either storage.
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration dnReg = dn.GetDNRegistrationForBP(bpid);
StorageBlockReport[] reports = new StorageBlockReport[cluster.GetStoragesPerDatanode
()];
AList <Replica> blocks = new AList <Replica>();
foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks())
{
Block localBlock = locatedBlock.GetBlock().GetLocalBlock();
blocks.AddItem(new FinalizedReplica(localBlock, null, null));
}
BlockListAsLongs bll = BlockListAsLongs.Encode(blocks);
for (int i = 0; i < cluster.GetStoragesPerDatanode(); ++i)
{
FsVolumeSpi v = dn.GetFSDataset().GetVolumes()[i];
DatanodeStorage dns = new DatanodeStorage(v.GetStorageID());
reports[i] = new StorageBlockReport(dns, bll);
}
// Should not assert!
cluster.GetNameNodeRpc().BlockReport(dnReg, bpid, reports, new BlockReportContext
(1, 0, Runtime.NanoTime()));
// Get the block locations once again.
locatedBlocks = client.GetLocatedBlocks(filename, 0, BlockSize * NumBlocks);
// Make sure that each block has two replicas, one on each DataNode.
foreach (LocatedBlock locatedBlock_1 in locatedBlocks.GetLocatedBlocks())
{
DatanodeInfo[] locations = locatedBlock_1.GetLocations();
Assert.AssertThat(locations.Length, IS.Is((int)NumDatanodes));
Assert.AssertThat(locations[0].GetDatanodeUuid(), CoreMatchers.Not(locations[1].GetDatanodeUuid
()));
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void VerifyIncrementalBlockReports(bool splitReports)
{
// Get the block list for the file with the block locations.
LocatedBlocks blocks = CreateFileGetBlocks(GenericTestUtils.GetMethodName());
// We will send 'fake' incremental block reports to the NN that look
// like they originated from DN 0.
StorageReceivedDeletedBlocks[] reports = new StorageReceivedDeletedBlocks[dn0.GetFSDataset
().GetVolumes().Count];
// Lie to the NN that one block on each storage has been deleted.
for (int i = 0; i < reports.Length; ++i)
{
FsVolumeSpi volume = dn0.GetFSDataset().GetVolumes()[i];
bool foundBlockOnStorage = false;
ReceivedDeletedBlockInfo[] rdbi = new ReceivedDeletedBlockInfo[1];
// Find the first block on this storage and mark it as deleted for the
// report.
foreach (LocatedBlock block in blocks.GetLocatedBlocks())
{
if (block.GetStorageIDs()[0].Equals(volume.GetStorageID()))
{
rdbi[0] = new ReceivedDeletedBlockInfo(block.GetBlock().GetLocalBlock(), ReceivedDeletedBlockInfo.BlockStatus
.DeletedBlock, null);
foundBlockOnStorage = true;
break;
}
}
NUnit.Framework.Assert.IsTrue(foundBlockOnStorage);
reports[i] = new StorageReceivedDeletedBlocks(volume.GetStorageID(), rdbi);
if (splitReports)
{
// If we are splitting reports then send the report for this storage now.
StorageReceivedDeletedBlocks[] singletonReport = new StorageReceivedDeletedBlocks
[] { reports[i] };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dn0Reg, poolId, singletonReport);
}
}
if (!splitReports)
{
// Send a combined report.
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dn0Reg, poolId, reports);
}
// Make sure that the deleted block from each storage was picked up
// by the NameNode.
Assert.AssertThat(cluster.GetNamesystem().GetMissingBlocksCount(), IS.Is((long)reports
.Length));
}
internal VolumeScanner(BlockScanner.Conf conf, DataNode datanode, FsVolumeReference
@ref)
{
this.conf = conf;
this.datanode = datanode;
this.@ref = @ref;
this.volume = @ref.GetVolume();
VolumeScanner.ScanResultHandler handler;
try
{
handler = System.Activator.CreateInstance(conf.resultHandler);
}
catch (Exception e)
{
Log.Error("unable to instantiate {}", conf.resultHandler, e);
handler = new VolumeScanner.ScanResultHandler();
}
this.resultHandler = handler;
SetName("VolumeScannerThread(" + volume.GetBasePath() + ")");
SetDaemon(true);
}
public virtual void Handle(ExtendedBlock block, IOException e)
{
FsVolumeSpi volume = scanner.volume;
if (e == null)
{
Log.Trace("Successfully scanned {} on {}", block, volume.GetBasePath());
return;
}
// If the block does not exist anymore, then it's not an error.
if (!volume.GetDataset().Contains(block))
{
Log.Debug("Volume {}: block {} is no longer in the dataset.", volume.GetBasePath(
), block);
return;
}
// If the block exists, the exception may due to a race with write:
// The BlockSender got an old block path in rbw. BlockReceiver removed
// the rbw block from rbw to finalized but BlockSender tried to open the
// file before BlockReceiver updated the VolumeMap. The state of the
// block can be changed again now, so ignore this error here. If there
// is a block really deleted by mistake, DirectoryScan should catch it.
if (e is FileNotFoundException)
{
Log.Info("Volume {}: verification failed for {} because of " + "FileNotFoundException. This may be due to a race with write."
, volume.GetBasePath(), block);
return;
}
Log.Warn("Reporting bad {} on {}", block, volume.GetBasePath());
try
{
scanner.datanode.ReportBadBlocks(block);
}
catch (IOException)
{
// This is bad, but not bad enough to shut down the scanner.
Log.Warn("Cannot report bad " + block.GetBlockId(), e);
}
}
/// <summary>Set up a scanner for the given block pool and volume.</summary>
/// <param name="ref">A reference to the volume.</param>
public virtual void AddVolumeScanner(FsVolumeReference @ref)
{
lock (this)
{
bool success = false;
try
{
FsVolumeSpi volume = @ref.GetVolume();
if (!IsEnabled())
{
Log.Debug("Not adding volume scanner for {}, because the block " + "scanner is disabled."
, volume.GetBasePath());
return;
}
VolumeScanner scanner = scanners[volume.GetStorageID()];
if (scanner != null)
{
Log.Error("Already have a scanner for volume {}.", volume.GetBasePath());
return;
}
Log.Debug("Adding scanner for volume {} (StorageID {})", volume.GetBasePath(), volume
.GetStorageID());
scanner = new VolumeScanner(conf, datanode, @ref);
scanner.Start();
scanners[volume.GetStorageID()] = scanner;
success = true;
}
finally
{
if (!success)
{
// If we didn't create a new VolumeScanner object, we don't
// need this reference to the volume.
IOUtils.Cleanup(null, @ref);
}
}
}
}
/// <summary>
/// Stops and removes a volume scanner.<p/>
/// This function will block until the volume scanner has stopped.
/// </summary>
/// <param name="volume">The volume to remove.</param>
public virtual void RemoveVolumeScanner(FsVolumeSpi volume)
{
lock (this)
{
if (!IsEnabled())
{
Log.Debug("Not removing volume scanner for {}, because the block " + "scanner is disabled."
, volume.GetStorageID());
return;
}
VolumeScanner scanner = scanners[volume.GetStorageID()];
if (scanner == null)
{
Log.Warn("No scanner found to remove for volumeId {}", volume.GetStorageID());
return;
}
Log.Info("Removing scanner for volume {} (StorageID {})", volume.GetBasePath(), volume
.GetStorageID());
scanner.Shutdown();
Sharpen.Collections.Remove(scanners, volume.GetStorageID());
Uninterruptibles.JoinUninterruptibly(scanner, 5, TimeUnit.Minutes);
}
}
/// <summary>Constructor</summary>
/// <param name="block">a block</param>
/// <param name="vol">volume where replica is located</param>
/// <param name="dir">directory path where block and meta files are located</param>
/// <param name="writer">a thread that is writing to this replica</param>
public ReplicaBeingWritten(Block block, FsVolumeSpi vol, FilePath dir, Sharpen.Thread
writer)
: base(block, vol, dir, writer)
{
}
/// <summary>Constructor for a zero length replica</summary>
/// <param name="blockId">block id</param>
/// <param name="genStamp">replica generation stamp</param>
/// <param name="vol">volume where replica is located</param>
/// <param name="dir">directory path where block and meta files are located</param>
/// <param name="bytesToReserve">
/// disk space to reserve for this replica, based on
/// the estimated maximum block length.
/// </param>
public ReplicaBeingWritten(long blockId, long genStamp, FsVolumeSpi vol, FilePath
dir, long bytesToReserve)
: base(blockId, genStamp, vol, dir, bytesToReserve)
{
}
/// <summary>Constructor</summary>
/// <param name="block">a block</param>
/// <param name="vol">volume where replica is located</param>
/// <param name="dir">directory path where block and meta files are located</param>
public FinalizedReplica(Block block, FsVolumeSpi vol, FilePath dir)
: base(block, vol, dir)
{
}
/// <summary>
/// Test iterating through a bunch of blocks in a volume using a volume
/// iterator.<p/>
/// We will rewind the iterator when about halfway through the blocks.
/// </summary>
/// <param name="numFiles">The number of files to create.</param>
/// <param name="maxStaleness">The maximum staleness to allow with the iterator.</param>
/// <exception cref="System.Exception"/>
private void TestVolumeIteratorImpl(int numFiles, long maxStaleness)
{
Configuration conf = new Configuration();
DisableBlockScanner(conf);
TestBlockScanner.TestContext ctx = new TestBlockScanner.TestContext(conf, 1);
ctx.CreateFiles(0, numFiles, 1);
NUnit.Framework.Assert.AreEqual(1, ctx.volumes.Count);
FsVolumeSpi volume = ctx.volumes[0];
ExtendedBlock savedBlock = null;
ExtendedBlock loadedBlock = null;
bool testedRewind = false;
bool testedSave = false;
bool testedLoad = false;
int blocksProcessed = 0;
int savedBlocksProcessed = 0;
try
{
BPOfferService[] bpos = ctx.datanode.GetAllBpOs();
NUnit.Framework.Assert.AreEqual(1, bpos.Length);
FsVolumeSpi.BlockIterator iter = volume.NewBlockIterator(ctx.bpids[0], "test");
NUnit.Framework.Assert.AreEqual(ctx.bpids[0], iter.GetBlockPoolId());
iter.SetMaxStalenessMs(maxStaleness);
while (true)
{
HashSet <ExtendedBlock> blocks = new HashSet <ExtendedBlock>();
for (int blockIdx = 0; blockIdx < numFiles; blockIdx++)
{
blocks.AddItem(ctx.GetFileBlock(0, blockIdx));
}
while (true)
{
ExtendedBlock block = iter.NextBlock();
if (block == null)
{
break;
}
blocksProcessed++;
Log.Info("BlockIterator for {} found block {}, blocksProcessed = {}", volume, block
, blocksProcessed);
if (testedSave && (savedBlock == null))
{
savedBlock = block;
}
if (testedLoad && (loadedBlock == null))
{
loadedBlock = block;
// The block that we get back right after loading the iterator
// should be the same block we got back right after saving
// the iterator.
NUnit.Framework.Assert.AreEqual(savedBlock, loadedBlock);
}
bool blockRemoved = blocks.Remove(block);
NUnit.Framework.Assert.IsTrue("Found unknown block " + block, blockRemoved);
if (blocksProcessed > (numFiles / 3))
{
if (!testedSave)
{
Log.Info("Processed {} blocks out of {}. Saving iterator.", blocksProcessed, numFiles
);
iter.Save();
testedSave = true;
savedBlocksProcessed = blocksProcessed;
}
}
if (blocksProcessed > (numFiles / 2))
{
if (!testedRewind)
{
Log.Info("Processed {} blocks out of {}. Rewinding iterator.", blocksProcessed,
numFiles);
iter.Rewind();
break;
}
}
if (blocksProcessed > ((2 * numFiles) / 3))
{
if (!testedLoad)
{
Log.Info("Processed {} blocks out of {}. Loading iterator.", blocksProcessed, numFiles
);
iter = volume.LoadBlockIterator(ctx.bpids[0], "test");
iter.SetMaxStalenessMs(maxStaleness);
break;
}
}
}
if (!testedRewind)
{
testedRewind = true;
blocksProcessed = 0;
Log.Info("Starting again at the beginning...");
continue;
}
if (!testedLoad)
{
testedLoad = true;
blocksProcessed = savedBlocksProcessed;
Log.Info("Starting again at the load point...");
continue;
}
NUnit.Framework.Assert.AreEqual(numFiles, blocksProcessed);
break;
}
}
finally
{
ctx.Close();
}
}
/// <summary>Constructor for a zero length replica</summary>
/// <param name="blockId">block id</param>
/// <param name="genStamp">replica generation stamp</param>
/// <param name="vol">volume where replica is located</param>
/// <param name="dir">directory path where block and meta files are located</param>
/// <param name="bytesToReserve">
/// disk space to reserve for this replica, based on
/// the estimated maximum block length.
/// </param>
public ReplicaInPipeline(long blockId, long genStamp, FsVolumeSpi vol, FilePath dir
, long bytesToReserve)
: this(blockId, 0L, genStamp, vol, dir, Sharpen.Thread.CurrentThread(), bytesToReserve
)
{
}
/// <summary>Constructor</summary>
/// <param name="blockId">block id</param>
/// <param name="len">replica length</param>
/// <param name="genStamp">replica generation stamp</param>
/// <param name="vol">volume where replica is located</param>
/// <param name="dir">directory path where block and meta files are located</param>
public ReplicaWaitingToBeRecovered(long blockId, long len, long genStamp, FsVolumeSpi
vol, FilePath dir)
: base(blockId, len, genStamp, vol, dir)
{
}
/// <summary>Set the volume where this replica is located on disk</summary>
internal virtual void SetVolume(FsVolumeSpi vol)
{
this.volume = vol;
}
public ReportCompiler(DataNode datanode, FsVolumeSpi volume)
{
this.datanode = datanode;
this.volume = volume;
}
public override void CheckAndUpdate(string bpid, long blockId, FilePath diskFile,
FilePath diskMetaFile, FsVolumeSpi vol)
{
}
/// <summary>
/// Compile list
/// <see cref="ScanInfo"/>
/// for the blocks in the directory <dir>
/// </summary>
private List <DirectoryScanner.ScanInfo> CompileReport(FsVolumeSpi vol, FilePath bpFinalizedDir
, FilePath dir, List <DirectoryScanner.ScanInfo> report)
{
FilePath[] files;
try
{
files = FileUtil.ListFiles(dir);
}
catch (IOException ioe)
{
Log.Warn("Exception occured while compiling report: ", ioe);
// Initiate a check on disk failure.
datanode.CheckDiskErrorAsync();
// Ignore this directory and proceed.
return(report);
}
Arrays.Sort(files);
/*
* Assumption: In the sorted list of files block file appears immediately
* before block metadata file. This is true for the current naming
* convention for block file blk_<blockid> and meta file
* blk_<blockid>_<genstamp>.meta
*/
for (int i = 0; i < files.Length; i++)
{
if (files[i].IsDirectory())
{
CompileReport(vol, bpFinalizedDir, files[i], report);
continue;
}
if (!Block.IsBlockFilename(files[i]))
{
if (IsBlockMetaFile(Block.BlockFilePrefix, files[i].GetName()))
{
long blockId = Block.GetBlockId(files[i].GetName());
VerifyFileLocation(files[i].GetParentFile(), bpFinalizedDir, blockId);
report.AddItem(new DirectoryScanner.ScanInfo(blockId, null, files[i], vol));
}
continue;
}
FilePath blockFile = files[i];
long blockId_1 = Block.Filename2id(blockFile.GetName());
FilePath metaFile = null;
// Skip all the files that start with block name until
// getting to the metafile for the block
while (i + 1 < files.Length && files[i + 1].IsFile() && files[i + 1].GetName().StartsWith
(blockFile.GetName()))
{
i++;
if (IsBlockMetaFile(blockFile.GetName(), files[i].GetName()))
{
metaFile = files[i];
break;
}
}
VerifyFileLocation(blockFile.GetParentFile(), bpFinalizedDir, blockId_1);
report.AddItem(new DirectoryScanner.ScanInfo(blockId_1, blockFile, metaFile, vol)
);
}
return(report);
}
/// <summary>Constructor</summary>
/// <param name="block">a block</param>
/// <param name="vol">volume where replica is located</param>
/// <param name="dir">directory path where block and meta files are located</param>
/// <param name="writer">a thread that is writing to this replica</param>
internal ReplicaInPipeline(Block block, FsVolumeSpi vol, FilePath dir, Sharpen.Thread
writer)
: this(block.GetBlockId(), block.GetNumBytes(), block.GetGenerationStamp(), vol,
dir, writer, 0L)
{
}
/// <summary>Constructor</summary>
/// <param name="blockId">block id</param>
/// <param name="len">replica length</param>
/// <param name="genStamp">replica generation stamp</param>
/// <param name="vol">volume where replica is located</param>
/// <param name="dir">directory path where block and meta files are located</param>
/// <param name="writer">a thread that is writing to this replica</param>
/// <param name="bytesToReserve">
/// disk space to reserve for this replica, based on
/// the estimated maximum block length.
/// </param>
public ReplicaBeingWritten(long blockId, long len, long genStamp, FsVolumeSpi vol
, FilePath dir, Sharpen.Thread writer, long bytesToReserve)
: base(blockId, len, genStamp, vol, dir, writer, bytesToReserve)
{
}
/// <summary>Constructor</summary>
/// <param name="blockId">block id</param>
/// <param name="len">replica length</param>
/// <param name="genStamp">replica generation stamp</param>
/// <param name="vol">volume where replica is located</param>
/// <param name="dir">directory path where block and meta files are located</param>
public FinalizedReplica(long blockId, long len, long genStamp, FsVolumeSpi vol, FilePath
dir)
: base(blockId, len, genStamp, vol, dir)
{
}
/// <summary>Constructor</summary>
/// <param name="block">a block</param>
/// <param name="vol">volume where replica is located</param>
/// <param name="dir">directory path where block and meta files are located</param>
public ReplicaWaitingToBeRecovered(Block block, FsVolumeSpi vol, FilePath dir)
: base(block, vol, dir)
{
}
/// <summary>Constructor</summary>
/// <param name="block">a block</param>
/// <param name="vol">volume where replica is located</param>
/// <param name="dir">directory path where block and meta files are located</param>
internal ReplicaInfo(Block block, FsVolumeSpi vol, FilePath dir)
: this(block.GetBlockId(), block.GetNumBytes(), block.GetGenerationStamp(), vol,
dir)
{
}
internal override void SetVolume(FsVolumeSpi vol)
{
//ReplicaInfo
base.SetVolume(vol);
original.SetVolume(vol);
}
