/// <summary>Truncate a block file</summary> /// <exception cref="System.IO.IOException"/> private long TruncateBlockFile() { lock (fds) { foreach (ReplicaInfo b in FsDatasetTestUtil.GetReplicas(fds, bpid)) { FilePath f = b.GetBlockFile(); FilePath mf = b.GetMetaFile(); // Truncate a block file that has a corresponding metadata file if (f.Exists() && f.Length() != 0 && mf.Exists()) { FileOutputStream s = null; FileChannel channel = null; try { s = new FileOutputStream(f); channel = s.GetChannel(); channel.Truncate(0); Log.Info("Truncated block file " + f.GetAbsolutePath()); return(b.GetBlockId()); } finally { IOUtils.Cleanup(Log, channel, s); } } } } return(0); }
/// <summary>Duplicate the given block on all volumes.</summary> /// <param name="blockId"/> /// <exception cref="System.IO.IOException"/> private void DuplicateBlock(long blockId) { lock (fds) { ReplicaInfo b = FsDatasetTestUtil.FetchReplicaInfo(fds, bpid, blockId); foreach (FsVolumeSpi v in fds.GetVolumes()) { if (v.GetStorageID().Equals(b.GetVolume().GetStorageID())) { continue; } // Volume without a copy of the block. Make a copy now. FilePath sourceBlock = b.GetBlockFile(); FilePath sourceMeta = b.GetMetaFile(); string sourceRoot = b.GetVolume().GetBasePath(); string destRoot = v.GetBasePath(); string relativeBlockPath = new FilePath(sourceRoot).ToURI().Relativize(sourceBlock .ToURI()).GetPath(); string relativeMetaPath = new FilePath(sourceRoot).ToURI().Relativize(sourceMeta. ToURI()).GetPath(); FilePath destBlock = new FilePath(destRoot, relativeBlockPath); FilePath destMeta = new FilePath(destRoot, relativeMetaPath); destBlock.GetParentFile().Mkdirs(); FileUtils.CopyFile(sourceBlock, destBlock); FileUtils.CopyFile(sourceMeta, destMeta); if (destBlock.Exists() && destMeta.Exists()) { Log.Info("Copied " + sourceBlock + " ==> " + destBlock); Log.Info("Copied " + sourceMeta + " ==> " + destMeta); } } } }
private void VerifyGenStamp(long blockId, long genStamp) { ReplicaInfo memBlock; memBlock = FsDatasetTestUtil.FetchReplicaInfo(fds, bpid, blockId); NUnit.Framework.Assert.IsNotNull(memBlock); NUnit.Framework.Assert.AreEqual(genStamp, memBlock.GetGenerationStamp()); }
private void VerifyStorageType(long blockId, bool expectTransient) { ReplicaInfo memBlock; memBlock = FsDatasetTestUtil.FetchReplicaInfo(fds, bpid, blockId); NUnit.Framework.Assert.IsNotNull(memBlock); MatcherAssert.AssertThat(memBlock.GetVolume().IsTransientStorage(), IS.Is(expectTransient )); }
/// <summary> /// Asserts that the storage lock file in each given directory has been /// released. /// </summary> /// <remarks> /// Asserts that the storage lock file in each given directory has been /// released. This method works by trying to acquire the lock file itself. If /// locking fails here, then the main code must have failed to release it. /// </remarks> /// <param name="dirs">every storage directory to check</param> /// <exception cref="System.IO.IOException">if there is an unexpected I/O error</exception> private static void AssertFileLocksReleased(ICollection <string> dirs) { foreach (string dir in dirs) { try { FsDatasetTestUtil.AssertFileLockReleased(dir); } catch (IOException e) { Log.Warn(e); } } }
/// <summary>Get a random blockId that is not used already</summary> private long GetFreeBlockId() { long id = rand.NextLong(); while (true) { id = rand.NextLong(); if (FsDatasetTestUtil.FetchReplicaInfo(fds, bpid, id) == null) { break; } } return(id); }
private void VerifyAddition(long blockId, long genStamp, long size) { ReplicaInfo replicainfo; replicainfo = FsDatasetTestUtil.FetchReplicaInfo(fds, bpid, blockId); NUnit.Framework.Assert.IsNotNull(replicainfo); // Added block has the same file as the one created by the test FilePath file = new FilePath(GetBlockFile(blockId)); NUnit.Framework.Assert.AreEqual(file.GetName(), FsDatasetTestUtil.GetFile(fds, bpid , blockId).GetName()); // Generation stamp is same as that of created file NUnit.Framework.Assert.AreEqual(genStamp, replicainfo.GetGenerationStamp()); // File size matches NUnit.Framework.Assert.AreEqual(size, replicainfo.GetNumBytes()); }
/// <exception cref="System.Exception"/> private static ReplicaInPipeline GetReplica(DataNode datanode, string bpid, HdfsServerConstants.ReplicaState expectedState) { ICollection <ReplicaInfo> replicas = FsDatasetTestUtil.GetReplicas(datanode.GetFSDataset (), bpid); for (int i = 0; i < 5 && replicas.Count == 0; i++) { Log.Info("wait since replicas.size() == 0; i=" + i); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.AreEqual(1, replicas.Count); ReplicaInfo r = replicas.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(expectedState, r.GetState()); return((ReplicaInPipeline)r); }
/// <summary>Delete block meta file</summary> private long DeleteMetaFile() { lock (fds) { foreach (ReplicaInfo b in FsDatasetTestUtil.GetReplicas(fds, bpid)) { FilePath file = b.GetMetaFile(); // Delete a metadata file if (file.Exists() && file.Delete()) { Log.Info("Deleting metadata file " + file.GetAbsolutePath()); return(b.GetBlockId()); } } } return(0); }
/// <summary>Delete a block file</summary> private long DeleteBlockFile() { lock (fds) { foreach (ReplicaInfo b in FsDatasetTestUtil.GetReplicas(fds, bpid)) { FilePath f = b.GetBlockFile(); FilePath mf = b.GetMetaFile(); // Delete a block file that has corresponding metadata file if (f.Exists() && mf.Exists() && f.Delete()) { Log.Info("Deleting block file " + f.GetAbsolutePath()); return(b.GetBlockId()); } } } return(0); }
/// <exception cref="System.Exception"/> public virtual void TestVerifyBlockChecksumCommand() { DFSTestUtil.CreateFile(fs, new Path("/bar"), 1234, (short)1, unchecked ((int)(0xdeadbeef ))); FsDatasetSpi <object> fsd = datanode.GetFSDataset(); ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, new Path("/bar")); FilePath blockFile = FsDatasetTestUtil.GetBlockFile(fsd, block.GetBlockPoolId(), block.GetLocalBlock()); NUnit.Framework.Assert.AreEqual("ret: 1, You must specify a meta file with -meta" , RunCmd(new string[] { "verify", "-block", blockFile.GetAbsolutePath() })); FilePath metaFile = FsDatasetTestUtil.GetMetaFile(fsd, block.GetBlockPoolId(), block .GetLocalBlock()); NUnit.Framework.Assert.AreEqual("ret: 0, Checksum type: " + "DataChecksum(type=CRC32C, chunkSize=512)" , RunCmd(new string[] { "verify", "-meta", metaFile.GetAbsolutePath() })); NUnit.Framework.Assert.AreEqual("ret: 0, Checksum type: " + "DataChecksum(type=CRC32C, chunkSize=512)" + "Checksum verification succeeded on block file " + blockFile.GetAbsolutePath( ), RunCmd(new string[] { "verify", "-meta", metaFile.GetAbsolutePath(), "-block" , blockFile.GetAbsolutePath() })); }
/// <exception cref="System.Exception"/> public virtual void TestDeleteBlockOnTransientStorage() { cluster = new MiniDFSCluster.Builder(Conf).StorageTypes(new StorageType[] { StorageType .RamDisk, StorageType.Default }).NumDataNodes(1).Build(); try { cluster.WaitActive(); bpid = cluster.GetNamesystem().GetBlockPoolId(); DataNode dataNode = cluster.GetDataNodes()[0]; fds = DataNodeTestUtils.GetFSDataset(cluster.GetDataNodes()[0]); client = cluster.GetFileSystem().GetClient(); scanner = new DirectoryScanner(dataNode, fds, Conf); scanner.SetRetainDiffs(true); FsDatasetTestUtil.StopLazyWriter(cluster.GetDataNodes()[0]); // Create a file file on RAM_DISK IList <LocatedBlock> blocks = CreateFile(GenericTestUtils.GetMethodName(), BlockLength , true); // Ensure no difference between volumeMap and disk. Scan(1, 0, 0, 0, 0, 0); // Make a copy of the block on DEFAULT storage and ensure that it is // picked up by the scanner. DuplicateBlock(blocks[0].GetBlock().GetBlockId()); Scan(2, 1, 0, 0, 0, 0, 1); // Ensure that the copy on RAM_DISK was deleted. VerifyStorageType(blocks[0].GetBlock().GetBlockId(), false); Scan(1, 0, 0, 0, 0, 0); } finally { if (scanner != null) { scanner.Shutdown(); scanner = null; } cluster.Shutdown(); cluster = null; } }
public static FilePath GetFile(DataNode dn, string bpid, long bid) { return(FsDatasetTestUtil.GetFile(dn.GetFSDataset(), bpid, bid)); }
public static long GetPendingAsyncDeletions(DataNode dn) { return(FsDatasetTestUtil.GetPendingAsyncDeletions(dn.GetFSDataset())); }
/// <exception cref="System.IO.IOException"/> public static FilePath GetMetaFile(DataNode dn, string bpid, Block b) { return(FsDatasetTestUtil.GetMetaFile(dn.GetFSDataset(), bpid, b)); }
/// <exception cref="System.IO.IOException"/> public static bool UnlinkBlock(DataNode dn, ExtendedBlock bk, int numLinks) { return(FsDatasetTestUtil.UnlinkBlock(dn.GetFSDataset(), bk, numLinks)); }
private void VerifyDeletion(long blockId) { // Ensure block does not exist in memory NUnit.Framework.Assert.IsNull(FsDatasetTestUtil.FetchReplicaInfo(fds, bpid, blockId )); }
/// <summary>Fetch a copy of ReplicaInfo from a datanode by block id</summary> /// <param name="dn">datanode to retrieve a replicainfo object from</param> /// <param name="bpid">Block pool Id</param> /// <param name="blkId">id of the replica's block</param> /// <returns>copy of ReplicaInfo object @link{FSDataset#fetchReplicaInfo}</returns> public static ReplicaInfo FetchReplicaInfo(DataNode dn, string bpid, long blkId) { return(FsDatasetTestUtil.FetchReplicaInfo(dn.GetFSDataset(), bpid, blkId)); }
/// <summary> /// Test that DataStorage and BlockPoolSliceStorage remove the failed volume /// after failure. /// </summary> /// <exception cref="System.Exception"/> /// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.TimeoutException"/> public virtual void TestFailedVolumeBeingRemovedFromDataNode() { Path file1 = new Path("/test1"); DFSTestUtil.CreateFile(fs, file1, 1024, (short)2, 1L); DFSTestUtil.WaitReplication(fs, file1, (short)2); FilePath dn0Vol1 = new FilePath(dataDir, "data" + (2 * 0 + 1)); DataNodeTestUtils.InjectDataDirFailure(dn0Vol1); DataNode dn0 = cluster.GetDataNodes()[0]; long lastDiskErrorCheck = dn0.GetLastDiskErrorCheck(); dn0.CheckDiskErrorAsync(); // Wait checkDiskError thread finish to discover volume failure. while (dn0.GetLastDiskErrorCheck() == lastDiskErrorCheck) { Sharpen.Thread.Sleep(100); } // Verify dn0Vol1 has been completely removed from DN0. // 1. dn0Vol1 is removed from DataStorage. DataStorage storage = dn0.GetStorage(); NUnit.Framework.Assert.AreEqual(1, storage.GetNumStorageDirs()); for (int i = 0; i < storage.GetNumStorageDirs(); i++) { Storage.StorageDirectory sd = storage.GetStorageDir(i); NUnit.Framework.Assert.IsFalse(sd.GetRoot().GetAbsolutePath().StartsWith(dn0Vol1. GetAbsolutePath())); } string bpid = cluster.GetNamesystem().GetBlockPoolId(); BlockPoolSliceStorage bpsStorage = storage.GetBPStorage(bpid); NUnit.Framework.Assert.AreEqual(1, bpsStorage.GetNumStorageDirs()); for (int i_1 = 0; i_1 < bpsStorage.GetNumStorageDirs(); i_1++) { Storage.StorageDirectory sd = bpsStorage.GetStorageDir(i_1); NUnit.Framework.Assert.IsFalse(sd.GetRoot().GetAbsolutePath().StartsWith(dn0Vol1. GetAbsolutePath())); } // 2. dn0Vol1 is removed from FsDataset FsDatasetSpi <FsVolumeSpi> data = dn0.GetFSDataset(); foreach (FsVolumeSpi volume in data.GetVolumes()) { Assert.AssertNotEquals(new FilePath(volume.GetBasePath()).GetAbsoluteFile(), dn0Vol1 .GetAbsoluteFile()); } // 3. all blocks on dn0Vol1 have been removed. foreach (ReplicaInfo replica in FsDatasetTestUtil.GetReplicas(data, bpid)) { NUnit.Framework.Assert.IsNotNull(replica.GetVolume()); Assert.AssertNotEquals(new FilePath(replica.GetVolume().GetBasePath()).GetAbsoluteFile (), dn0Vol1.GetAbsoluteFile()); } // 4. dn0Vol1 is not in DN0's configuration and dataDirs anymore. string[] dataDirStrs = dn0.GetConf().Get(DFSConfigKeys.DfsDatanodeDataDirKey).Split (","); NUnit.Framework.Assert.AreEqual(1, dataDirStrs.Length); NUnit.Framework.Assert.IsFalse(dataDirStrs[0].Contains(dn0Vol1.GetAbsolutePath()) ); }