public virtual void TestGet()
{
// Test 1: null argument throws invalid argument exception
try
{
map.Get(bpid, null);
NUnit.Framework.Assert.Fail("Expected exception not thrown");
}
catch (ArgumentException)
{
}
// Test 2: successful lookup based on block
NUnit.Framework.Assert.IsNotNull(map.Get(bpid, block));
// Test 3: Lookup failure - generation stamp mismatch
Block b = new Block(block);
b.SetGenerationStamp(0);
NUnit.Framework.Assert.IsNull(map.Get(bpid, b));
// Test 4: Lookup failure - blockID mismatch
b.SetGenerationStamp(block.GetGenerationStamp());
b.SetBlockId(0);
NUnit.Framework.Assert.IsNull(map.Get(bpid, b));
// Test 5: successful lookup based on block ID
NUnit.Framework.Assert.IsNotNull(map.Get(bpid, block.GetBlockId()));
// Test 6: failed lookup for invalid block ID
NUnit.Framework.Assert.IsNull(map.Get(bpid, 0));
}
public virtual void TestInitReplicaRecovery()
{
long firstblockid = 10000L;
long gs = 7777L;
long length = 22L;
ReplicaMap map = new ReplicaMap(this);
string bpid = "BP-TEST";
Block[] blocks = new Block[5];
for (int i = 0; i < blocks.Length; i++)
{
blocks[i] = new Block(firstblockid + i, length, gs);
map.Add(bpid, CreateReplicaInfo(blocks[i]));
}
{
//normal case
Block b = blocks[0];
ReplicaInfo originalInfo = map.Get(bpid, b);
long recoveryid = gs + 1;
ReplicaRecoveryInfo recoveryInfo = FsDatasetImpl.InitReplicaRecovery(bpid, map, blocks
[0], recoveryid, DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisDefault);
AssertEquals(originalInfo, recoveryInfo);
ReplicaUnderRecovery updatedInfo = (ReplicaUnderRecovery)map.Get(bpid, b);
NUnit.Framework.Assert.AreEqual(originalInfo.GetBlockId(), updatedInfo.GetBlockId
());
NUnit.Framework.Assert.AreEqual(recoveryid, updatedInfo.GetRecoveryID());
//recover one more time
long recoveryid2 = gs + 2;
ReplicaRecoveryInfo recoveryInfo2 = FsDatasetImpl.InitReplicaRecovery(bpid, map,
blocks[0], recoveryid2, DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisDefault
);
AssertEquals(originalInfo, recoveryInfo2);
ReplicaUnderRecovery updatedInfo2 = (ReplicaUnderRecovery)map.Get(bpid, b);
NUnit.Framework.Assert.AreEqual(originalInfo.GetBlockId(), updatedInfo2.GetBlockId
());
NUnit.Framework.Assert.AreEqual(recoveryid2, updatedInfo2.GetRecoveryID());
//case RecoveryInProgressException
try
{
FsDatasetImpl.InitReplicaRecovery(bpid, map, b, recoveryid, DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisDefault
);
NUnit.Framework.Assert.Fail();
}
catch (RecoveryInProgressException ripe)
{
System.Console.Out.WriteLine("GOOD: getting " + ripe);
}
}
{
// BlockRecoveryFI_01: replica not found
long recoveryid = gs + 1;
Block b = new Block(firstblockid - 1, length, gs);
ReplicaRecoveryInfo r = FsDatasetImpl.InitReplicaRecovery(bpid, map, b, recoveryid
, DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisDefault);
NUnit.Framework.Assert.IsNull("Data-node should not have this replica.", r);
}
{
// BlockRecoveryFI_02: "THIS IS NOT SUPPOSED TO HAPPEN" with recovery id < gs
long recoveryid = gs - 1;
Block b = new Block(firstblockid + 1, length, gs);
try
{
FsDatasetImpl.InitReplicaRecovery(bpid, map, b, recoveryid, DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisDefault
);
NUnit.Framework.Assert.Fail();
}
catch (IOException ioe)
{
System.Console.Out.WriteLine("GOOD: getting " + ioe);
}
}
{
// BlockRecoveryFI_03: Replica's gs is less than the block's gs
long recoveryid = gs + 1;
Block b = new Block(firstblockid, length, gs + 1);
try
{
FsDatasetImpl.InitReplicaRecovery(bpid, map, b, recoveryid, DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisDefault
);
NUnit.Framework.Assert.Fail("InitReplicaRecovery should fail because replica's "
+ "gs is less than the block's gs");
}
catch (IOException e)
{
e.Message.StartsWith("replica.getGenerationStamp() < block.getGenerationStamp(), block="
);
}
}
}
/// <summary>Add replicas under the given directory to the volume map</summary>
/// <param name="volumeMap">the replicas map</param>
/// <param name="dir">an input directory</param>
/// <param name="lazyWriteReplicaMap">
/// Map of replicas on transient
/// storage.
/// </param>
/// <param name="isFinalized">
/// true if the directory has finalized replicas;
/// false if the directory has rbw replicas
/// </param>
/// <exception cref="System.IO.IOException"/>
internal virtual void AddToReplicasMap(ReplicaMap volumeMap, FilePath dir, RamDiskReplicaTracker
lazyWriteReplicaMap, bool isFinalized)
{
FilePath[] files = FileUtil.ListFiles(dir);
foreach (FilePath file in files)
{
if (file.IsDirectory())
{
AddToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized);
}
if (isFinalized && FsDatasetUtil.IsUnlinkTmpFile(file))
{
file = RecoverTempUnlinkedBlock(file);
if (file == null)
{
// the original block still exists, so we cover it
// in another iteration and can continue here
continue;
}
}
if (!Block.IsBlockFilename(file))
{
continue;
}
long genStamp = FsDatasetUtil.GetGenerationStampFromFile(files, file);
long blockId = Block.Filename2id(file.GetName());
ReplicaInfo newReplica = null;
if (isFinalized)
{
newReplica = new FinalizedReplica(blockId, file.Length(), genStamp, volume, file.
GetParentFile());
}
else
{
bool loadRwr = true;
FilePath restartMeta = new FilePath(file.GetParent() + FilePath.pathSeparator + "."
+ file.GetName() + ".restart");
Scanner sc = null;
try
{
sc = new Scanner(restartMeta, "UTF-8");
// The restart meta file exists
if (sc.HasNextLong() && (sc.NextLong() > Time.Now()))
{
// It didn't expire. Load the replica as a RBW.
// We don't know the expected block length, so just use 0
// and don't reserve any more space for writes.
newReplica = new ReplicaBeingWritten(blockId, ValidateIntegrityAndSetLength(file,
genStamp), genStamp, volume, file.GetParentFile(), null, 0);
loadRwr = false;
}
sc.Close();
if (!restartMeta.Delete())
{
FsDatasetImpl.Log.Warn("Failed to delete restart meta file: " + restartMeta.GetPath
());
}
}
catch (FileNotFoundException)
{
}
finally
{
// nothing to do hereFile dir =
if (sc != null)
{
sc.Close();
}
}
// Restart meta doesn't exist or expired.
if (loadRwr)
{
newReplica = new ReplicaWaitingToBeRecovered(blockId, ValidateIntegrityAndSetLength
(file, genStamp), genStamp, volume, file.GetParentFile());
}
}
ReplicaInfo oldReplica = volumeMap.Get(bpid, newReplica.GetBlockId());
if (oldReplica == null)
{
volumeMap.Add(bpid, newReplica);
}
else
{
// We have multiple replicas of the same block so decide which one
// to keep.
newReplica = ResolveDuplicateReplicas(newReplica, oldReplica, volumeMap);
}
// If we are retaining a replica on transient storage make sure
// it is in the lazyWriteReplicaMap so it can be persisted
// eventually.
if (newReplica.GetVolume().IsTransientStorage())
{
lazyWriteReplicaMap.AddReplica(bpid, blockId, (FsVolumeImpl)newReplica.GetVolume(
));
}
else
{
lazyWriteReplicaMap.DiscardReplica(bpid, blockId, false);
}
}
}