public virtual void TestDuplicateReplicaResolution()
{
FsVolumeImpl fsv1 = Org.Mockito.Mockito.Mock <FsVolumeImpl>();
FsVolumeImpl fsv2 = Org.Mockito.Mockito.Mock <FsVolumeImpl>();
FilePath f1 = new FilePath("d1/block");
FilePath f2 = new FilePath("d2/block");
ReplicaInfo replicaOlder = new FinalizedReplica(1, 1, 1, fsv1, f1);
ReplicaInfo replica = new FinalizedReplica(1, 2, 2, fsv1, f1);
ReplicaInfo replicaSame = new FinalizedReplica(1, 2, 2, fsv1, f1);
ReplicaInfo replicaNewer = new FinalizedReplica(1, 3, 3, fsv1, f1);
ReplicaInfo replicaOtherOlder = new FinalizedReplica(1, 1, 1, fsv2, f2);
ReplicaInfo replicaOtherSame = new FinalizedReplica(1, 2, 2, fsv2, f2);
ReplicaInfo replicaOtherNewer = new FinalizedReplica(1, 3, 3, fsv2, f2);
// equivalent path so don't remove either
NUnit.Framework.Assert.IsNull(BlockPoolSlice.SelectReplicaToDelete(replicaSame, replica
));
NUnit.Framework.Assert.IsNull(BlockPoolSlice.SelectReplicaToDelete(replicaOlder,
replica));
NUnit.Framework.Assert.IsNull(BlockPoolSlice.SelectReplicaToDelete(replicaNewer,
replica));
// keep latest found replica
NUnit.Framework.Assert.AreSame(replica, BlockPoolSlice.SelectReplicaToDelete(replicaOtherSame
, replica));
NUnit.Framework.Assert.AreSame(replicaOtherOlder, BlockPoolSlice.SelectReplicaToDelete
(replicaOtherOlder, replica));
NUnit.Framework.Assert.AreSame(replica, BlockPoolSlice.SelectReplicaToDelete(replicaOtherNewer
, replica));
}
public virtual void TestDeletingBlocks()
{
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).Build
();
try
{
cluster.WaitActive();
DataNode dn = cluster.GetDataNodes()[0];
FsDatasetImpl ds = (FsDatasetImpl)DataNodeTestUtils.GetFSDataset(dn);
FsVolumeImpl vol = ds.GetVolumes()[0];
ExtendedBlock eb;
ReplicaInfo info;
IList <Block> blockList = new AList <Block>();
for (int i = 1; i <= 63; i++)
{
eb = new ExtendedBlock(Blockpool, i, 1, 1000 + i);
info = new FinalizedReplica(eb.GetLocalBlock(), vol, vol.GetCurrentDir().GetParentFile
());
ds.volumeMap.Add(Blockpool, info);
info.GetBlockFile().CreateNewFile();
info.GetMetaFile().CreateNewFile();
blockList.AddItem(info);
}
ds.Invalidate(Blockpool, Sharpen.Collections.ToArray(blockList, new Block[0]));
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
// Nothing to do
NUnit.Framework.Assert.IsTrue(ds.IsDeletingBlock(Blockpool, blockList[0].GetBlockId
()));
blockList.Clear();
eb = new ExtendedBlock(Blockpool, 64, 1, 1064);
info = new FinalizedReplica(eb.GetLocalBlock(), vol, vol.GetCurrentDir().GetParentFile
());
ds.volumeMap.Add(Blockpool, info);
info.GetBlockFile().CreateNewFile();
info.GetMetaFile().CreateNewFile();
blockList.AddItem(info);
ds.Invalidate(Blockpool, Sharpen.Collections.ToArray(blockList, new Block[0]));
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
// Nothing to do
NUnit.Framework.Assert.IsFalse(ds.IsDeletingBlock(Blockpool, blockList[0].GetBlockId
()));
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Generate testing environment and return a collection of blocks
/// on which to run the tests.
/// </summary>
/// <param name="bpid">Block pool ID to generate blocks for</param>
/// <param name="dataSet">Namespace in which to insert blocks</param>
/// <returns>Contrived blocks for further testing.</returns>
/// <exception cref="System.IO.IOException"/>
private ExtendedBlock[] Setup(string bpid, FsDatasetImpl dataSet)
{
// setup replicas map
ExtendedBlock[] blocks = new ExtendedBlock[] { new ExtendedBlock(bpid, 1, 1, 2001
), new ExtendedBlock(bpid, 2, 1, 2002), new ExtendedBlock(bpid, 3, 1, 2003), new
ExtendedBlock(bpid, 4, 1, 2004), new ExtendedBlock(bpid, 5, 1, 2005), new ExtendedBlock
(bpid, 6, 1, 2006) };
ReplicaMap replicasMap = dataSet.volumeMap;
FsVolumeImpl vol = (FsVolumeImpl)dataSet.volumes.GetNextVolume(StorageType.Default
, 0).GetVolume();
ReplicaInfo replicaInfo = new FinalizedReplica(blocks[Finalized].GetLocalBlock(),
vol, vol.GetCurrentDir().GetParentFile());
replicasMap.Add(bpid, replicaInfo);
replicaInfo.GetBlockFile().CreateNewFile();
replicaInfo.GetMetaFile().CreateNewFile();
replicasMap.Add(bpid, new ReplicaInPipeline(blocks[Temporary].GetBlockId(), blocks
[Temporary].GetGenerationStamp(), vol, vol.CreateTmpFile(bpid, blocks[Temporary]
.GetLocalBlock()).GetParentFile(), 0));
replicaInfo = new ReplicaBeingWritten(blocks[Rbw].GetLocalBlock(), vol, vol.CreateRbwFile
(bpid, blocks[Rbw].GetLocalBlock()).GetParentFile(), null);
replicasMap.Add(bpid, replicaInfo);
replicaInfo.GetBlockFile().CreateNewFile();
replicaInfo.GetMetaFile().CreateNewFile();
replicasMap.Add(bpid, new ReplicaWaitingToBeRecovered(blocks[Rwr].GetLocalBlock()
, vol, vol.CreateRbwFile(bpid, blocks[Rwr].GetLocalBlock()).GetParentFile()));
replicasMap.Add(bpid, new ReplicaUnderRecovery(new FinalizedReplica(blocks[Rur].GetLocalBlock
(), vol, vol.GetCurrentDir().GetParentFile()), 2007));
return(blocks);
}
public virtual void TestFuzz()
{
Replica[] replicas = new Replica[100000];
Random rand = new Random(0);
for (int i = 0; i < replicas.Length; i++)
{
Block b = new Block(rand.NextLong(), i, i << 4);
switch (rand.Next(2))
{
case 0:
{
replicas[i] = new FinalizedReplica(b, null, null);
break;
}
case 1:
{
replicas[i] = new ReplicaBeingWritten(b, null, null, null);
break;
}
case 2:
{
replicas[i] = new ReplicaWaitingToBeRecovered(b, null, null);
break;
}
}
}
CheckReport(replicas);
}
public virtual void TestDatanodeDetect()
{
AtomicReference <DatanodeProtocolProtos.BlockReportRequestProto> request = new AtomicReference
<DatanodeProtocolProtos.BlockReportRequestProto>();
// just capture the outgoing PB
DatanodeProtocolPB mockProxy = Org.Mockito.Mockito.Mock <DatanodeProtocolPB>();
Org.Mockito.Mockito.DoAnswer(new _Answer_205(request)).When(mockProxy).BlockReport
(Matchers.Any <RpcController>(), Matchers.Any <DatanodeProtocolProtos.BlockReportRequestProto
>());
DatanodeProtocolClientSideTranslatorPB nn = new DatanodeProtocolClientSideTranslatorPB
(mockProxy);
DatanodeRegistration reg = DFSTestUtil.GetLocalDatanodeRegistration();
NamespaceInfo nsInfo = new NamespaceInfo(1, "cluster", "bp", 1);
reg.SetNamespaceInfo(nsInfo);
Replica r = new FinalizedReplica(new Block(1, 2, 3), null, null);
BlockListAsLongs bbl = BlockListAsLongs.Encode(Sharpen.Collections.Singleton(r));
DatanodeStorage storage = new DatanodeStorage("s1");
StorageBlockReport[] sbr = new StorageBlockReport[] { new StorageBlockReport(storage
, bbl) };
// check DN sends new-style BR
request.Set(null);
nsInfo.SetCapabilities(NamespaceInfo.Capability.StorageBlockReportBuffers.GetMask
());
nn.BlockReport(reg, "pool", sbr, new BlockReportContext(1, 0, Runtime.NanoTime())
);
DatanodeProtocolProtos.BlockReportRequestProto proto = request.Get();
NUnit.Framework.Assert.IsNotNull(proto);
NUnit.Framework.Assert.IsTrue(proto.GetReports(0).GetBlocksList().IsEmpty());
NUnit.Framework.Assert.IsFalse(proto.GetReports(0).GetBlocksBuffersList().IsEmpty
());
// back up to prior version and check DN sends old-style BR
request.Set(null);
nsInfo.SetCapabilities(NamespaceInfo.Capability.Unknown.GetMask());
nn.BlockReport(reg, "pool", sbr, new BlockReportContext(1, 0, Runtime.NanoTime())
);
proto = request.Get();
NUnit.Framework.Assert.IsNotNull(proto);
NUnit.Framework.Assert.IsFalse(proto.GetReports(0).GetBlocksList().IsEmpty());
NUnit.Framework.Assert.IsTrue(proto.GetReports(0).GetBlocksBuffersList().IsEmpty(
));
}
/// <summary>Add replicas under the given directory to the volume map</summary>
/// <param name="volumeMap">the replicas map</param>
/// <param name="dir">an input directory</param>
/// <param name="lazyWriteReplicaMap">
/// Map of replicas on transient
/// storage.
/// </param>
/// <param name="isFinalized">
/// true if the directory has finalized replicas;
/// false if the directory has rbw replicas
/// </param>
/// <exception cref="System.IO.IOException"/>
internal virtual void AddToReplicasMap(ReplicaMap volumeMap, FilePath dir, RamDiskReplicaTracker
lazyWriteReplicaMap, bool isFinalized)
{
FilePath[] files = FileUtil.ListFiles(dir);
foreach (FilePath file in files)
{
if (file.IsDirectory())
{
AddToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized);
}
if (isFinalized && FsDatasetUtil.IsUnlinkTmpFile(file))
{
file = RecoverTempUnlinkedBlock(file);
if (file == null)
{
// the original block still exists, so we cover it
// in another iteration and can continue here
continue;
}
}
if (!Block.IsBlockFilename(file))
{
continue;
}
long genStamp = FsDatasetUtil.GetGenerationStampFromFile(files, file);
long blockId = Block.Filename2id(file.GetName());
ReplicaInfo newReplica = null;
if (isFinalized)
{
newReplica = new FinalizedReplica(blockId, file.Length(), genStamp, volume, file.
GetParentFile());
}
else
{
bool loadRwr = true;
FilePath restartMeta = new FilePath(file.GetParent() + FilePath.pathSeparator + "."
+ file.GetName() + ".restart");
Scanner sc = null;
try
{
sc = new Scanner(restartMeta, "UTF-8");
// The restart meta file exists
if (sc.HasNextLong() && (sc.NextLong() > Time.Now()))
{
// It didn't expire. Load the replica as a RBW.
// We don't know the expected block length, so just use 0
// and don't reserve any more space for writes.
newReplica = new ReplicaBeingWritten(blockId, ValidateIntegrityAndSetLength(file,
genStamp), genStamp, volume, file.GetParentFile(), null, 0);
loadRwr = false;
}
sc.Close();
if (!restartMeta.Delete())
{
FsDatasetImpl.Log.Warn("Failed to delete restart meta file: " + restartMeta.GetPath
());
}
}
catch (FileNotFoundException)
{
}
finally
{
// nothing to do hereFile dir =
if (sc != null)
{
sc.Close();
}
}
// Restart meta doesn't exist or expired.
if (loadRwr)
{
newReplica = new ReplicaWaitingToBeRecovered(blockId, ValidateIntegrityAndSetLength
(file, genStamp), genStamp, volume, file.GetParentFile());
}
}
ReplicaInfo oldReplica = volumeMap.Get(bpid, newReplica.GetBlockId());
if (oldReplica == null)
{
volumeMap.Add(bpid, newReplica);
}
else
{
// We have multiple replicas of the same block so decide which one
// to keep.
newReplica = ResolveDuplicateReplicas(newReplica, oldReplica, volumeMap);
}
// If we are retaining a replica on transient storage make sure
// it is in the lazyWriteReplicaMap so it can be persisted
// eventually.
if (newReplica.GetVolume().IsTransientStorage())
{
lazyWriteReplicaMap.AddReplica(bpid, blockId, (FsVolumeImpl)newReplica.GetVolume(
));
}
else
{
lazyWriteReplicaMap.DiscardReplica(bpid, blockId, false);
}
}
}
