public virtual void TestGet() { // Test 1: null argument throws invalid argument exception try { map.Get(bpid, null); NUnit.Framework.Assert.Fail("Expected exception not thrown"); } catch (ArgumentException) { } // Test 2: successful lookup based on block NUnit.Framework.Assert.IsNotNull(map.Get(bpid, block)); // Test 3: Lookup failure - generation stamp mismatch Block b = new Block(block); b.SetGenerationStamp(0); NUnit.Framework.Assert.IsNull(map.Get(bpid, b)); // Test 4: Lookup failure - blockID mismatch b.SetGenerationStamp(block.GetGenerationStamp()); b.SetBlockId(0); NUnit.Framework.Assert.IsNull(map.Get(bpid, b)); // Test 5: successful lookup based on block ID NUnit.Framework.Assert.IsNotNull(map.Get(bpid, block.GetBlockId())); // Test 6: failed lookup for invalid block ID NUnit.Framework.Assert.IsNull(map.Get(bpid, 0)); }
public virtual void TestInitReplicaRecovery() { long firstblockid = 10000L; long gs = 7777L; long length = 22L; ReplicaMap map = new ReplicaMap(this); string bpid = "BP-TEST"; Block[] blocks = new Block[5]; for (int i = 0; i < blocks.Length; i++) { blocks[i] = new Block(firstblockid + i, length, gs); map.Add(bpid, CreateReplicaInfo(blocks[i])); } { //normal case Block b = blocks[0]; ReplicaInfo originalInfo = map.Get(bpid, b); long recoveryid = gs + 1; ReplicaRecoveryInfo recoveryInfo = FsDatasetImpl.InitReplicaRecovery(bpid, map, blocks [0], recoveryid, DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisDefault); AssertEquals(originalInfo, recoveryInfo); ReplicaUnderRecovery updatedInfo = (ReplicaUnderRecovery)map.Get(bpid, b); NUnit.Framework.Assert.AreEqual(originalInfo.GetBlockId(), updatedInfo.GetBlockId ()); NUnit.Framework.Assert.AreEqual(recoveryid, updatedInfo.GetRecoveryID()); //recover one more time long recoveryid2 = gs + 2; ReplicaRecoveryInfo recoveryInfo2 = FsDatasetImpl.InitReplicaRecovery(bpid, map, blocks[0], recoveryid2, DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisDefault ); AssertEquals(originalInfo, recoveryInfo2); ReplicaUnderRecovery updatedInfo2 = (ReplicaUnderRecovery)map.Get(bpid, b); NUnit.Framework.Assert.AreEqual(originalInfo.GetBlockId(), updatedInfo2.GetBlockId ()); NUnit.Framework.Assert.AreEqual(recoveryid2, updatedInfo2.GetRecoveryID()); //case RecoveryInProgressException try { FsDatasetImpl.InitReplicaRecovery(bpid, map, b, recoveryid, DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisDefault ); NUnit.Framework.Assert.Fail(); } catch (RecoveryInProgressException ripe) { System.Console.Out.WriteLine("GOOD: getting " + ripe); } } { // BlockRecoveryFI_01: replica not found long recoveryid = gs + 1; Block b = new Block(firstblockid - 1, length, gs); ReplicaRecoveryInfo r = FsDatasetImpl.InitReplicaRecovery(bpid, map, b, recoveryid , DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisDefault); NUnit.Framework.Assert.IsNull("Data-node should not have this replica.", r); } { // BlockRecoveryFI_02: "THIS IS NOT SUPPOSED TO HAPPEN" with recovery id < gs long recoveryid = gs - 1; Block b = new Block(firstblockid + 1, length, gs); try { FsDatasetImpl.InitReplicaRecovery(bpid, map, b, recoveryid, DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisDefault ); NUnit.Framework.Assert.Fail(); } catch (IOException ioe) { System.Console.Out.WriteLine("GOOD: getting " + ioe); } } { // BlockRecoveryFI_03: Replica's gs is less than the block's gs long recoveryid = gs + 1; Block b = new Block(firstblockid, length, gs + 1); try { FsDatasetImpl.InitReplicaRecovery(bpid, map, b, recoveryid, DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisDefault ); NUnit.Framework.Assert.Fail("InitReplicaRecovery should fail because replica's " + "gs is less than the block's gs"); } catch (IOException e) { e.Message.StartsWith("replica.getGenerationStamp() < block.getGenerationStamp(), block=" ); } } }
/// <summary>Add replicas under the given directory to the volume map</summary> /// <param name="volumeMap">the replicas map</param> /// <param name="dir">an input directory</param> /// <param name="lazyWriteReplicaMap"> /// Map of replicas on transient /// storage. /// </param> /// <param name="isFinalized"> /// true if the directory has finalized replicas; /// false if the directory has rbw replicas /// </param> /// <exception cref="System.IO.IOException"/> internal virtual void AddToReplicasMap(ReplicaMap volumeMap, FilePath dir, RamDiskReplicaTracker lazyWriteReplicaMap, bool isFinalized) { FilePath[] files = FileUtil.ListFiles(dir); foreach (FilePath file in files) { if (file.IsDirectory()) { AddToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized); } if (isFinalized && FsDatasetUtil.IsUnlinkTmpFile(file)) { file = RecoverTempUnlinkedBlock(file); if (file == null) { // the original block still exists, so we cover it // in another iteration and can continue here continue; } } if (!Block.IsBlockFilename(file)) { continue; } long genStamp = FsDatasetUtil.GetGenerationStampFromFile(files, file); long blockId = Block.Filename2id(file.GetName()); ReplicaInfo newReplica = null; if (isFinalized) { newReplica = new FinalizedReplica(blockId, file.Length(), genStamp, volume, file. GetParentFile()); } else { bool loadRwr = true; FilePath restartMeta = new FilePath(file.GetParent() + FilePath.pathSeparator + "." + file.GetName() + ".restart"); Scanner sc = null; try { sc = new Scanner(restartMeta, "UTF-8"); // The restart meta file exists if (sc.HasNextLong() && (sc.NextLong() > Time.Now())) { // It didn't expire. Load the replica as a RBW. // We don't know the expected block length, so just use 0 // and don't reserve any more space for writes. newReplica = new ReplicaBeingWritten(blockId, ValidateIntegrityAndSetLength(file, genStamp), genStamp, volume, file.GetParentFile(), null, 0); loadRwr = false; } sc.Close(); if (!restartMeta.Delete()) { FsDatasetImpl.Log.Warn("Failed to delete restart meta file: " + restartMeta.GetPath ()); } } catch (FileNotFoundException) { } finally { // nothing to do hereFile dir = if (sc != null) { sc.Close(); } } // Restart meta doesn't exist or expired. if (loadRwr) { newReplica = new ReplicaWaitingToBeRecovered(blockId, ValidateIntegrityAndSetLength (file, genStamp), genStamp, volume, file.GetParentFile()); } } ReplicaInfo oldReplica = volumeMap.Get(bpid, newReplica.GetBlockId()); if (oldReplica == null) { volumeMap.Add(bpid, newReplica); } else { // We have multiple replicas of the same block so decide which one // to keep. newReplica = ResolveDuplicateReplicas(newReplica, oldReplica, volumeMap); } // If we are retaining a replica on transient storage make sure // it is in the lazyWriteReplicaMap so it can be persisted // eventually. if (newReplica.GetVolume().IsTransientStorage()) { lazyWriteReplicaMap.AddReplica(bpid, blockId, (FsVolumeImpl)newReplica.GetVolume( )); } else { lazyWriteReplicaMap.DiscardReplica(bpid, blockId, false); } } }