public virtual void TestAbandonBlock() { string src = FileNamePrefix + "foo"; // Start writing a file but do not close it FSDataOutputStream fout = fs.Create(new Path(src), true, 4096, (short)1, 512L); for (int i = 0; i < 1024; i++) { fout.Write(123); } fout.Hflush(); long fileId = ((DFSOutputStream)fout.GetWrappedStream()).GetFileId(); // Now abandon the last block DFSClient dfsclient = DFSClientAdapter.GetDFSClient(fs); LocatedBlocks blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue ); int orginalNumBlocks = blocks.LocatedBlockCount(); LocatedBlock b = blocks.GetLastLocatedBlock(); dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName ); // call abandonBlock again to make sure the operation is idempotent dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName ); // And close the file fout.Close(); // Close cluster and check the block has been abandoned after restart cluster.RestartNameNode(); blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue); NUnit.Framework.Assert.AreEqual("Blocks " + b + " has not been abandoned.", orginalNumBlocks , blocks.LocatedBlockCount() + 1); }
// Waits for all of the blocks to have expected replication // Waits for all of the blocks to have expected replication /// <exception cref="System.IO.IOException"/> private void WaitForBlockReplication(string filename, ClientProtocol namenode, int expected, long maxWaitSec) { long start = Time.MonotonicNow(); //wait for all the blocks to be replicated; Log.Info("Checking for block replication for " + filename); LocatedBlocks blocks = namenode.GetBlockLocations(filename, 0, long.MaxValue); NUnit.Framework.Assert.AreEqual(numBlocks, blocks.LocatedBlockCount()); for (int i = 0; i < numBlocks; ++i) { Log.Info("Checking for block:" + (i + 1)); while (true) { // Loop to check for block i (usually when 0 is done all will be done blocks = namenode.GetBlockLocations(filename, 0, long.MaxValue); NUnit.Framework.Assert.AreEqual(numBlocks, blocks.LocatedBlockCount()); LocatedBlock block = blocks.Get(i); int actual = block.GetLocations().Length; if (actual == expected) { Log.Info("Got enough replicas for " + (i + 1) + "th block " + block.GetBlock() + ", got " + actual + "."); break; } Log.Info("Not enough replicas for " + (i + 1) + "th block " + block.GetBlock() + " yet. Expecting " + expected + ", got " + actual + "."); if (maxWaitSec > 0 && (Time.MonotonicNow() - start) > (maxWaitSec * 1000)) { throw new IOException("Timedout while waiting for all blocks to " + " be replicated for " + filename); } try { Sharpen.Thread.Sleep(500); } catch (Exception) { } } } }
public virtual void TestReplicationError() { // create a file of replication factor of 1 Path fileName = new Path("/test.txt"); int fileLen = 1; DFSTestUtil.CreateFile(fs, fileName, 1, (short)1, 1L); DFSTestUtil.WaitReplication(fs, fileName, (short)1); // get the block belonged to the created file LocatedBlocks blocks = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), fileName .ToString(), 0, (long)fileLen); NUnit.Framework.Assert.AreEqual("Should only find 1 block", blocks.LocatedBlockCount (), 1); LocatedBlock block = blocks.Get(0); // bring up a second datanode cluster.StartDataNodes(conf, 1, true, null, null); cluster.WaitActive(); int sndNode = 1; DataNode datanode = cluster.GetDataNodes()[sndNode]; // replicate the block to the second datanode IPEndPoint target = datanode.GetXferAddress(); Socket s = Sharpen.Extensions.CreateSocket(target.Address, target.Port); // write the header. DataOutputStream @out = new DataOutputStream(s.GetOutputStream()); DataChecksum checksum = DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, 512 ); new Sender(@out).WriteBlock(block.GetBlock(), StorageType.Default, BlockTokenSecretManager .DummyToken, string.Empty, new DatanodeInfo[0], new StorageType[0], null, BlockConstructionStage .PipelineSetupCreate, 1, 0L, 0L, 0L, checksum, CachingStrategy.NewDefaultStrategy (), false, false, null); @out.Flush(); // close the connection before sending the content of the block @out.Close(); // the temporary block & meta files should be deleted string bpid = cluster.GetNamesystem().GetBlockPoolId(); FilePath storageDir = cluster.GetInstanceStorageDir(sndNode, 0); FilePath dir1 = MiniDFSCluster.GetRbwDir(storageDir, bpid); storageDir = cluster.GetInstanceStorageDir(sndNode, 1); FilePath dir2 = MiniDFSCluster.GetRbwDir(storageDir, bpid); while (dir1.ListFiles().Length != 0 || dir2.ListFiles().Length != 0) { Sharpen.Thread.Sleep(100); } // then increase the file's replication factor fs.SetReplication(fileName, (short)2); // replication should succeed DFSTestUtil.WaitReplication(fs, fileName, (short)1); // clean up the file fs.Delete(fileName, false); }
/// <summary>TC11: Racing rename</summary> /// <exception cref="System.Exception"/> private void TestTC11(bool appendToNewBlock) { Path p = new Path("/TC11/foo" + (appendToNewBlock ? "0" : "1")); System.Console.Out.WriteLine("p=" + p); //a. Create file and write one block of data. Close file. int len1 = (int)BlockSize; { FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } //b. Reopen file in "append" mode. Append half block of data. FSDataOutputStream out_1 = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag. Append, CreateFlag.NewBlock), 4096, null) : fs.Append(p); int len2 = (int)BlockSize / 2; AppendTestUtil.Write(out_1, len1, len2); out_1.Hflush(); //c. Rename file to file.new. Path pnew = new Path(p + ".new"); NUnit.Framework.Assert.IsTrue(fs.Rename(p, pnew)); //d. Close file handle that was opened in (b). out_1.Close(); //check block sizes long len = fs.GetFileStatus(pnew).GetLen(); LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(pnew.ToString (), 0L, len); int numblock = locatedblocks.LocatedBlockCount(); for (int i = 0; i < numblock; i++) { LocatedBlock lb = locatedblocks.Get(i); ExtendedBlock blk = lb.GetBlock(); long size = lb.GetBlockSize(); if (i < numblock - 1) { NUnit.Framework.Assert.AreEqual(BlockSize, size); } foreach (DatanodeInfo datanodeinfo in lb.GetLocations()) { DataNode dn = cluster.GetDataNode(datanodeinfo.GetIpcPort()); Block metainfo = DataNodeTestUtils.GetFSDataset(dn).GetStoredBlock(blk.GetBlockPoolId (), blk.GetBlockId()); NUnit.Framework.Assert.AreEqual(size, metainfo.GetNumBytes()); } } }
/// <summary>TC7: Corrupted replicas are present.</summary> /// <exception cref="System.IO.IOException">an exception might be thrown</exception> /// <exception cref="System.Exception"/> private void TestTC7(bool appendToNewBlock) { short repl = 2; Path p = new Path("/TC7/foo" + (appendToNewBlock ? "0" : "1")); System.Console.Out.WriteLine("p=" + p); //a. Create file with replication factor of 2. Write half block of data. Close file. int len1 = (int)(BlockSize / 2); { FSDataOutputStream @out = fs.Create(p, false, buffersize, repl, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } DFSTestUtil.WaitReplication(fs, p, repl); //b. Log into one datanode that has one replica of this block. // Find the block file on this datanode and truncate it to zero size. LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(p.ToString() , 0L, len1); NUnit.Framework.Assert.AreEqual(1, locatedblocks.LocatedBlockCount()); LocatedBlock lb = locatedblocks.Get(0); ExtendedBlock blk = lb.GetBlock(); NUnit.Framework.Assert.AreEqual(len1, lb.GetBlockSize()); DatanodeInfo[] datanodeinfos = lb.GetLocations(); NUnit.Framework.Assert.AreEqual(repl, datanodeinfos.Length); DataNode dn = cluster.GetDataNode(datanodeinfos[0].GetIpcPort()); FilePath f = DataNodeTestUtils.GetBlockFile(dn, blk.GetBlockPoolId(), blk.GetLocalBlock ()); RandomAccessFile raf = new RandomAccessFile(f, "rw"); AppendTestUtil.Log.Info("dn=" + dn + ", blk=" + blk + " (length=" + blk.GetNumBytes () + ")"); NUnit.Framework.Assert.AreEqual(len1, raf.Length()); raf.SetLength(0); raf.Close(); //c. Open file in "append mode". Append a new block worth of data. Close file. int len2 = (int)BlockSize; { FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append , CreateFlag.NewBlock), 4096, null) : fs.Append(p); AppendTestUtil.Write(@out, len1, len2); @out.Close(); } //d. Reopen file and read two blocks worth of data. AppendTestUtil.Check(fs, p, len1 + len2); }
/* check if there are at least two nodes are on the same rack */ /// <exception cref="System.IO.IOException"/> private void CheckFile(FileSystem fileSys, Path name, int repl) { Configuration conf = fileSys.GetConf(); ClientProtocol namenode = NameNodeProxies.CreateProxy <ClientProtocol>(conf, fileSys .GetUri()).GetProxy(); WaitForBlockReplication(name.ToString(), namenode, Math.Min(numDatanodes, repl), -1); LocatedBlocks locations = namenode.GetBlockLocations(name.ToString(), 0, long.MaxValue ); FileStatus stat = fileSys.GetFileStatus(name); BlockLocation[] blockLocations = fileSys.GetFileBlockLocations(stat, 0L, long.MaxValue ); // verify that rack locations match NUnit.Framework.Assert.IsTrue(blockLocations.Length == locations.LocatedBlockCount ()); for (int i = 0; i < blockLocations.Length; i++) { LocatedBlock blk = locations.Get(i); DatanodeInfo[] datanodes = blk.GetLocations(); string[] topologyPaths = blockLocations[i].GetTopologyPaths(); NUnit.Framework.Assert.IsTrue(topologyPaths.Length == datanodes.Length); for (int j = 0; j < topologyPaths.Length; j++) { bool found = false; for (int k = 0; k < racks.Length; k++) { if (topologyPaths[j].StartsWith(racks[k])) { found = true; break; } } NUnit.Framework.Assert.IsTrue(found); } } bool isOnSameRack = true; bool isNotOnSameRack = true; foreach (LocatedBlock blk_1 in locations.GetLocatedBlocks()) { DatanodeInfo[] datanodes = blk_1.GetLocations(); if (datanodes.Length <= 1) { break; } if (datanodes.Length == 2) { isNotOnSameRack = !(datanodes[0].GetNetworkLocation().Equals(datanodes[1].GetNetworkLocation ())); break; } isOnSameRack = false; isNotOnSameRack = false; for (int i_1 = 0; i_1 < datanodes.Length - 1; i_1++) { Log.Info("datanode " + i_1 + ": " + datanodes[i_1]); bool onRack = false; for (int j = i_1 + 1; j < datanodes.Length; j++) { if (datanodes[i_1].GetNetworkLocation().Equals(datanodes[j].GetNetworkLocation())) { onRack = true; } } if (onRack) { isOnSameRack = true; } if (!onRack) { isNotOnSameRack = true; } if (isOnSameRack && isNotOnSameRack) { break; } } if (!isOnSameRack || !isNotOnSameRack) { break; } } NUnit.Framework.Assert.IsTrue(isOnSameRack); NUnit.Framework.Assert.IsTrue(isNotOnSameRack); }
/// <summary> /// Get a partial listing of the indicated directory /// We will stop when any of the following conditions is met: /// 1) this.lsLimit files have been added /// 2) needLocation is true AND enough files have been added such /// that at least this.lsLimit block locations are in the response /// </summary> /// <param name="fsd">FSDirectory</param> /// <param name="iip"> /// the INodesInPath instance containing all the INodes along the /// path /// </param> /// <param name="src">the directory name</param> /// <param name="startAfter">the name to start listing after</param> /// <param name="needLocation">if block locations are returned</param> /// <returns>a partial listing starting after startAfter</returns> /// <exception cref="System.IO.IOException"/> private static DirectoryListing GetListing(FSDirectory fsd, INodesInPath iip, string src, byte[] startAfter, bool needLocation, bool isSuperUser) { string srcs = FSDirectory.NormalizePath(src); bool isRawPath = FSDirectory.IsReservedRawName(src); fsd.ReadLock(); try { if (srcs.EndsWith(HdfsConstants.SeparatorDotSnapshotDir)) { return(GetSnapshotsListing(fsd, srcs, startAfter)); } int snapshot = iip.GetPathSnapshotId(); INode targetNode = iip.GetLastINode(); if (targetNode == null) { return(null); } byte parentStoragePolicy = isSuperUser ? targetNode.GetStoragePolicyID() : BlockStoragePolicySuite .IdUnspecified; if (!targetNode.IsDirectory()) { return(new DirectoryListing(new HdfsFileStatus[] { CreateFileStatus(fsd, src, HdfsFileStatus .EmptyName, targetNode, needLocation, parentStoragePolicy, snapshot, isRawPath, iip) }, 0)); } INodeDirectory dirInode = targetNode.AsDirectory(); ReadOnlyList <INode> contents = dirInode.GetChildrenList(snapshot); int startChild = INodeDirectory.NextChild(contents, startAfter); int totalNumChildren = contents.Size(); int numOfListing = Math.Min(totalNumChildren - startChild, fsd.GetLsLimit()); int locationBudget = fsd.GetLsLimit(); int listingCnt = 0; HdfsFileStatus[] listing = new HdfsFileStatus[numOfListing]; for (int i = 0; i < numOfListing && locationBudget > 0; i++) { INode cur = contents.Get(startChild + i); byte curPolicy = isSuperUser && !cur.IsSymlink() ? cur.GetLocalStoragePolicyID() : BlockStoragePolicySuite.IdUnspecified; listing[i] = CreateFileStatus(fsd, src, cur.GetLocalNameBytes(), cur, needLocation , GetStoragePolicyID(curPolicy, parentStoragePolicy), snapshot, isRawPath, iip); listingCnt++; if (needLocation) { // Once we hit lsLimit locations, stop. // This helps to prevent excessively large response payloads. // Approximate #locations with locatedBlockCount() * repl_factor LocatedBlocks blks = ((HdfsLocatedFileStatus)listing[i]).GetBlockLocations(); locationBudget -= (blks == null) ? 0 : blks.LocatedBlockCount() * listing[i].GetReplication (); } } // truncate return array if necessary if (listingCnt < numOfListing) { listing = Arrays.CopyOf(listing, listingCnt); } return(new DirectoryListing(listing, totalNumChildren - startChild - listingCnt)); } finally { fsd.ReadUnlock(); } }
public virtual void TestConcatNotCompleteBlock() { long trgFileLen = blockSize * 3; long srcFileLen = blockSize * 3 + 20; // block at the end - not full // create first file string name1 = "/trg"; string name2 = "/src"; Path filePath1 = new Path(name1); DFSTestUtil.CreateFile(dfs, filePath1, trgFileLen, ReplFactor, 1); HdfsFileStatus fStatus = nn.GetFileInfo(name1); long fileLen = fStatus.GetLen(); NUnit.Framework.Assert.AreEqual(fileLen, trgFileLen); //read the file FSDataInputStream stm = dfs.Open(filePath1); byte[] byteFile1 = new byte[(int)trgFileLen]; stm.ReadFully(0, byteFile1); stm.Close(); LocatedBlocks lb1 = nn.GetBlockLocations(name1, 0, trgFileLen); Path filePath2 = new Path(name2); DFSTestUtil.CreateFile(dfs, filePath2, srcFileLen, ReplFactor, 1); fStatus = nn.GetFileInfo(name2); fileLen = fStatus.GetLen(); NUnit.Framework.Assert.AreEqual(srcFileLen, fileLen); // read the file stm = dfs.Open(filePath2); byte[] byteFile2 = new byte[(int)srcFileLen]; stm.ReadFully(0, byteFile2); stm.Close(); LocatedBlocks lb2 = nn.GetBlockLocations(name2, 0, srcFileLen); System.Console.Out.WriteLine("trg len=" + trgFileLen + "; src len=" + srcFileLen); // move the blocks dfs.Concat(filePath1, new Path[] { filePath2 }); long totalLen = trgFileLen + srcFileLen; fStatus = nn.GetFileInfo(name1); fileLen = fStatus.GetLen(); // read the resulting file stm = dfs.Open(filePath1); byte[] byteFileConcat = new byte[(int)fileLen]; stm.ReadFully(0, byteFileConcat); stm.Close(); LocatedBlocks lbConcat = nn.GetBlockLocations(name1, 0, fileLen); //verifications // 1. number of blocks NUnit.Framework.Assert.AreEqual(lbConcat.LocatedBlockCount(), lb1.LocatedBlockCount () + lb2.LocatedBlockCount()); // 2. file lengths System.Console.Out.WriteLine("file1 len=" + fileLen + "; total len=" + totalLen); NUnit.Framework.Assert.AreEqual(fileLen, totalLen); // 3. removal of the src file fStatus = nn.GetFileInfo(name2); NUnit.Framework.Assert.IsNull("File " + name2 + "still exists", fStatus); // file shouldn't exist // 4. content CheckFileContent(byteFileConcat, new byte[][] { byteFile1, byteFile2 }); }