public bool Get() { try { LocatedBlocks locs = NameNodeAdapter.GetBlockLocations(nn, path, 0, 1000); DatanodeInfo[] dnis = locs.GetLastLocatedBlock().GetLocations(); foreach (DatanodeInfo dni in dnis) { NUnit.Framework.Assert.IsNotNull(dni); } int numReplicas = dnis.Length; Org.Apache.Hadoop.Hdfs.Server.Namenode.HA.TestStandbyIsHot.Log.Info("Got " + numReplicas + " locs: " + locs); if (numReplicas > expectedReplicas) { cluster.TriggerDeletionReports(); } cluster.TriggerHeartbeats(); return(numReplicas == expectedReplicas); } catch (IOException e) { Org.Apache.Hadoop.Hdfs.Server.Namenode.HA.TestStandbyIsHot.Log.Warn("No block locations yet: " + e.Message); return(false); } }
/// <summary>Create a redirection URL</summary> /// <exception cref="System.IO.IOException"/> private Uri CreateRedirectURL(string path, string encodedPath, HdfsFileStatus status , UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request, string dt) { string scheme = request.GetScheme(); LocatedBlocks blks = nnproxy.GetBlockLocations(status.GetFullPath(new Path(path)) .ToUri().GetPath(), 0, 1); Configuration conf = NameNodeHttpServer.GetConfFromContext(GetServletContext()); DatanodeID host = PickSrcDatanode(blks, status, conf); string hostname; if (host is DatanodeInfo) { hostname = host.GetHostName(); } else { hostname = host.GetIpAddr(); } int port = "https".Equals(scheme) ? host.GetInfoSecurePort() : host.GetInfoPort(); string dtParam = string.Empty; if (dt != null) { dtParam = JspHelper.GetDelegationTokenUrlParam(dt); } // Add namenode address to the url params NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext()); string addr = nn.GetNameNodeAddressHostPortString(); string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr); return(new Uri(scheme, hostname, port, "/streamFile" + encodedPath + '?' + "ugi=" + ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam)); }
public virtual void TestBlockTokenInLastLocatedBlock() { Configuration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsBlockAccessTokenEnableKey, true); conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, 512); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); try { FileSystem fs = cluster.GetFileSystem(); string fileName = "/testBlockTokenInLastLocatedBlock"; Path filePath = new Path(fileName); FSDataOutputStream @out = fs.Create(filePath, (short)1); @out.Write(new byte[1000]); // ensure that the first block is written out (see FSOutputSummer#flush) @out.Flush(); LocatedBlocks locatedBlocks = cluster.GetNameNodeRpc().GetBlockLocations(fileName , 0, 1000); while (locatedBlocks.GetLastLocatedBlock() == null) { Sharpen.Thread.Sleep(100); locatedBlocks = cluster.GetNameNodeRpc().GetBlockLocations(fileName, 0, 1000); } Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = locatedBlocks .GetLastLocatedBlock().GetBlockToken(); NUnit.Framework.Assert.AreEqual(BlockTokenIdentifier.KindName, token.GetKind()); @out.Close(); } finally { cluster.Shutdown(); } }
public virtual void TestGetBlockLocations() { NamenodeProtocols namenode = cluster.GetNameNodeRpc(); Path p = new Path(BaseDir, "file2.dat"); string src = p.ToString(); FSDataOutputStream @out = TestFileCreation.CreateFile(hdfs, p, 3); // write a half block int len = (int)(((uint)BlockSize) >> 1); WriteFile(p, @out, len); for (int i = 1; i < NumBlocks;) { // verify consistency LocatedBlocks lb = namenode.GetBlockLocations(src, 0, len); IList <LocatedBlock> blocks = lb.GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(i, blocks.Count); Block b = blocks[blocks.Count - 1].GetBlock().GetLocalBlock(); NUnit.Framework.Assert.IsTrue(b is BlockInfoContiguousUnderConstruction); if (++i < NumBlocks) { // write one more block WriteFile(p, @out, BlockSize); len += BlockSize; } } // close file @out.Close(); }
/// <summary>Create a file with one block and corrupt some/all of the block replicas. /// </summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/> /// <exception cref="System.IO.FileNotFoundException"/> /// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/> /// <exception cref="System.Exception"/> /// <exception cref="Sharpen.TimeoutException"/> private void CreateAFileWithCorruptedBlockReplicas(Path filePath, short repl, int corruptBlockCount) { DFSTestUtil.CreateFile(dfs, filePath, BlockSize, repl, 0); DFSTestUtil.WaitReplication(dfs, filePath, repl); // Locate the file blocks by asking name node LocatedBlocks locatedblocks = dfs.dfs.GetNamenode().GetBlockLocations(filePath.ToString (), 0L, BlockSize); NUnit.Framework.Assert.AreEqual(repl, locatedblocks.Get(0).GetLocations().Length); // The file only has one block LocatedBlock lblock = locatedblocks.Get(0); DatanodeInfo[] datanodeinfos = lblock.GetLocations(); ExtendedBlock block = lblock.GetBlock(); // corrupt some /all of the block replicas for (int i = 0; i < corruptBlockCount; i++) { DatanodeInfo dninfo = datanodeinfos[i]; DataNode dn = cluster.GetDataNode(dninfo.GetIpcPort()); CorruptBlock(block, dn); Log.Debug("Corrupted block " + block.GetBlockName() + " on data node " + dninfo); } }
/// <summary>TC12: Append to partial CRC chunk</summary> /// <exception cref="System.Exception"/> private void TestTC12(bool appendToNewBlock) { Path p = new Path("/TC12/foo" + (appendToNewBlock ? "0" : "1")); System.Console.Out.WriteLine("p=" + p); //a. Create file with a block size of 64KB // and a default io.bytes.per.checksum of 512 bytes. // Write 25687 bytes of data. Close file. int len1 = 25687; { FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } //b. Reopen file in "append" mode. Append another 5877 bytes of data. Close file. int len2 = 5877; { FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append , CreateFlag.NewBlock), 4096, null) : fs.Append(p); AppendTestUtil.Write(@out, len1, len2); @out.Close(); } //c. Reopen file and read 25687+5877 bytes of data from file. Close file. AppendTestUtil.Check(fs, p, len1 + len2); if (appendToNewBlock) { LocatedBlocks blks = fs.dfs.GetLocatedBlocks(p.ToString(), 0); NUnit.Framework.Assert.AreEqual(2, blks.GetLocatedBlocks().Count); NUnit.Framework.Assert.AreEqual(len1, blks.GetLocatedBlocks()[0].GetBlockSize()); NUnit.Framework.Assert.AreEqual(len2, blks.GetLocatedBlocks()[1].GetBlockSize()); AppendTestUtil.Check(fs, p, 0, len1); AppendTestUtil.Check(fs, p, len1, len2); } }
/// <exception cref="System.IO.IOException"/> public static DatanodeInfo BestNode(LocatedBlocks blks, Configuration conf) { Dictionary <DatanodeInfo, JspHelper.NodeRecord> map = new Dictionary <DatanodeInfo, JspHelper.NodeRecord>(); foreach (LocatedBlock block in blks.GetLocatedBlocks()) { DatanodeInfo[] nodes = block.GetLocations(); foreach (DatanodeInfo node in nodes) { JspHelper.NodeRecord record = map[node]; if (record == null) { map[node] = new JspHelper.NodeRecord(node, 1); } else { record.frequency++; } } } JspHelper.NodeRecord[] nodes_1 = Sharpen.Collections.ToArray(map.Values, new JspHelper.NodeRecord [map.Count]); Arrays.Sort(nodes_1, new JspHelper.NodeRecordComparator()); return(BestNode(nodes_1, false)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected internal bool VerifyDeletedBlocks(LocatedBlocks locatedBlocks) { Log.Info("Verifying replica has no saved copy after deletion."); TriggerBlockReport(); while (DataNodeTestUtils.GetPendingAsyncDeletions(cluster.GetDataNodes()[0]) > 0L ) { Sharpen.Thread.Sleep(1000); } string bpid = cluster.GetNamesystem().GetBlockPoolId(); IList <FsVolumeSpi> volumes = cluster.GetDataNodes()[0].GetFSDataset().GetVolumes( ); // Make sure deleted replica does not have a copy on either finalized dir of // transient volume or finalized dir of non-transient volume foreach (FsVolumeSpi v in volumes) { FsVolumeImpl volume = (FsVolumeImpl)v; FilePath targetDir = (v.IsTransientStorage()) ? volume.GetBlockPoolSlice(bpid).GetFinalizedDir () : volume.GetBlockPoolSlice(bpid).GetLazypersistDir(); if (VerifyBlockDeletedFromDir(targetDir, locatedBlocks) == false) { return(false); } } return(true); }
public virtual void TestFallbackToDiskPartial() { StartUpCluster(true, 2); string MethodName = GenericTestUtils.GetMethodName(); Path path = new Path("/" + MethodName + ".dat"); MakeTestFile(path, BlockSize * 5, true); // Sleep for a short time to allow the lazy writer thread to do its job Sharpen.Thread.Sleep(6 * LazyWriterIntervalSec * 1000); TriggerBlockReport(); int numBlocksOnRamDisk = 0; int numBlocksOnDisk = 0; long fileLength = client.GetFileInfo(path.ToString()).GetLen(); LocatedBlocks locatedBlocks = client.GetLocatedBlocks(path.ToString(), 0, fileLength ); foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks()) { if (locatedBlock.GetStorageTypes()[0] == StorageType.RamDisk) { numBlocksOnRamDisk++; } else { if (locatedBlock.GetStorageTypes()[0] == StorageType.Default) { numBlocksOnDisk++; } } } // Since eviction is asynchronous, depending on the timing of eviction // wrt writes, we may get 2 or less blocks on RAM disk. System.Diagnostics.Debug.Assert((numBlocksOnRamDisk <= 2)); System.Diagnostics.Debug.Assert((numBlocksOnDisk >= 3)); }
public virtual void TestLocatedBlocks2Locations() { DatanodeInfo d = DFSTestUtil.GetLocalDatanodeInfo(); DatanodeInfo[] ds = new DatanodeInfo[1]; ds[0] = d; // ok ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1); LocatedBlock l1 = new LocatedBlock(b1, ds, 0, false); // corrupt ExtendedBlock b2 = new ExtendedBlock("bpid", 2, 1, 1); LocatedBlock l2 = new LocatedBlock(b2, ds, 0, true); IList <LocatedBlock> ls = Arrays.AsList(l1, l2); LocatedBlocks lbs = new LocatedBlocks(10, false, ls, l2, true, null); BlockLocation[] bs = DFSUtil.LocatedBlocks2Locations(lbs); NUnit.Framework.Assert.IsTrue("expected 2 blocks but got " + bs.Length, bs.Length == 2); int corruptCount = 0; foreach (BlockLocation b in bs) { if (b.IsCorrupt()) { corruptCount++; } } NUnit.Framework.Assert.IsTrue("expected 1 corrupt files but got " + corruptCount, corruptCount == 1); // test an empty location bs = DFSUtil.LocatedBlocks2Locations(new LocatedBlocks()); NUnit.Framework.Assert.AreEqual(0, bs.Length); }
public virtual void TestAbandonBlock() { string src = FileNamePrefix + "foo"; // Start writing a file but do not close it FSDataOutputStream fout = fs.Create(new Path(src), true, 4096, (short)1, 512L); for (int i = 0; i < 1024; i++) { fout.Write(123); } fout.Hflush(); long fileId = ((DFSOutputStream)fout.GetWrappedStream()).GetFileId(); // Now abandon the last block DFSClient dfsclient = DFSClientAdapter.GetDFSClient(fs); LocatedBlocks blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue ); int orginalNumBlocks = blocks.LocatedBlockCount(); LocatedBlock b = blocks.GetLastLocatedBlock(); dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName ); // call abandonBlock again to make sure the operation is idempotent dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName ); // And close the file fout.Close(); // Close cluster and check the block has been abandoned after restart cluster.RestartNameNode(); blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue); NUnit.Framework.Assert.AreEqual("Blocks " + b + " has not been abandoned.", orginalNumBlocks , blocks.LocatedBlockCount() + 1); }
/// <summary>Create FileStatus with location info by file INode</summary> /// <exception cref="System.IO.IOException"/> private static HdfsLocatedFileStatus CreateLocatedFileStatus(FSDirectory fsd, string fullPath, byte[] path, INode node, byte storagePolicy, int snapshot, bool isRawPath , INodesInPath iip) { System.Diagnostics.Debug.Assert(fsd.HasReadLock()); long size = 0; // length is zero for directories short replication = 0; long blocksize = 0; LocatedBlocks loc = null; bool isEncrypted; FileEncryptionInfo feInfo = isRawPath ? null : fsd.GetFileEncryptionInfo(node, snapshot , iip); if (node.IsFile()) { INodeFile fileNode = node.AsFile(); size = fileNode.ComputeFileSize(snapshot); replication = fileNode.GetFileReplication(snapshot); blocksize = fileNode.GetPreferredBlockSize(); bool inSnapshot = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId; bool isUc = !inSnapshot && fileNode.IsUnderConstruction(); long fileSize = !inSnapshot && isUc?fileNode.ComputeFileSizeNotIncludingLastUcBlock () : size; loc = fsd.GetFSNamesystem().GetBlockManager().CreateLocatedBlocks(fileNode.GetBlocks (snapshot), fileSize, isUc, 0L, size, false, inSnapshot, feInfo); if (loc == null) { loc = new LocatedBlocks(); } isEncrypted = (feInfo != null) || (isRawPath && fsd.IsInAnEZ(INodesInPath.FromINode (node))); } else { isEncrypted = fsd.IsInAnEZ(INodesInPath.FromINode(node)); } int childrenNum = node.IsDirectory() ? node.AsDirectory().GetChildrenNum(snapshot ) : 0; INodeAttributes nodeAttrs = fsd.GetAttributes(fullPath, path, node, snapshot); HdfsLocatedFileStatus status = new HdfsLocatedFileStatus(size, node.IsDirectory() , replication, blocksize, node.GetModificationTime(snapshot), node.GetAccessTime (snapshot), GetPermissionForFileStatus(nodeAttrs, isEncrypted), nodeAttrs.GetUserName (), nodeAttrs.GetGroupName(), node.IsSymlink() ? node.AsSymlink().GetSymlink() : null, path, node.GetId(), loc, childrenNum, feInfo, storagePolicy); // Set caching information for the located blocks. if (loc != null) { CacheManager cacheManager = fsd.GetFSNamesystem().GetCacheManager(); foreach (LocatedBlock lb in loc.GetLocatedBlocks()) { cacheManager.SetCachedLocations(lb); } } return(status); }
/// <summary>Verify the first block of the file is corrupted (for all its replica).</summary> /// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/> /// <exception cref="System.IO.FileNotFoundException"/> /// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/> /// <exception cref="System.IO.IOException"/> private void VerifyFirstBlockCorrupted(Path filePath, bool isCorrupted) { LocatedBlocks locatedBlocks = dfs.dfs.GetNamenode().GetBlockLocations(filePath.ToUri ().GetPath(), 0, long.MaxValue); LocatedBlock firstLocatedBlock = locatedBlocks.Get(0); NUnit.Framework.Assert.AreEqual(isCorrupted, firstLocatedBlock.IsCorrupt()); }
/// <exception cref="System.IO.IOException"/> private LocatedBlock GetLocatedBlock() { LocatedBlocks locatedBlocks = client.GetLocatedBlocks(Path.ToString(), 0, BlockSize ); Assert.AssertThat(locatedBlocks.GetLocatedBlocks().Count, CoreMatchers.Is(1)); return(Iterables.GetOnlyElement(locatedBlocks.GetLocatedBlocks())); }
/// <summary> /// Regression test for HDFS-2795: /// - Start an HA cluster with a DN. /// </summary> /// <remarks> /// Regression test for HDFS-2795: /// - Start an HA cluster with a DN. /// - Write several blocks to the FS with replication 1. /// - Shutdown the DN /// - Wait for the NNs to declare the DN dead. All blocks will be under-replicated. /// - Restart the DN. /// In the bug, the standby node would only very slowly notice the blocks returning /// to the cluster. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestDatanodeRestarts() { Configuration conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, 1024); // We read from the standby to watch block locations HAUtil.SetAllowStandbyReads(conf, true); conf.SetLong(DFSConfigKeys.DfsNamenodeAccesstimePrecisionKey, 0); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology .SimpleHATopology()).NumDataNodes(1).Build(); try { NameNode nn0 = cluster.GetNameNode(0); NameNode nn1 = cluster.GetNameNode(1); cluster.TransitionToActive(0); // Create 5 blocks. DFSTestUtil.CreateFile(cluster.GetFileSystem(0), TestFilePath, 5 * 1024, (short)1 , 1L); HATestUtil.WaitForStandbyToCatchUp(nn0, nn1); // Stop the DN. DataNode dn = cluster.GetDataNodes()[0]; string dnName = dn.GetDatanodeId().GetXferAddr(); MiniDFSCluster.DataNodeProperties dnProps = cluster.StopDataNode(0); // Make sure both NNs register it as dead. BlockManagerTestUtil.NoticeDeadDatanode(nn0, dnName); BlockManagerTestUtil.NoticeDeadDatanode(nn1, dnName); BlockManagerTestUtil.UpdateState(nn0.GetNamesystem().GetBlockManager()); BlockManagerTestUtil.UpdateState(nn1.GetNamesystem().GetBlockManager()); NUnit.Framework.Assert.AreEqual(5, nn0.GetNamesystem().GetUnderReplicatedBlocks() ); // The SBN will not have any blocks in its neededReplication queue // since the SBN doesn't process replication. NUnit.Framework.Assert.AreEqual(0, nn1.GetNamesystem().GetUnderReplicatedBlocks() ); LocatedBlocks locs = nn1.GetRpcServer().GetBlockLocations(TestFile, 0, 1); NUnit.Framework.Assert.AreEqual("Standby should have registered that the block has no replicas" , 0, locs.Get(0).GetLocations().Length); cluster.RestartDataNode(dnProps); // Wait for both NNs to re-register the DN. cluster.WaitActive(0); cluster.WaitActive(1); BlockManagerTestUtil.UpdateState(nn0.GetNamesystem().GetBlockManager()); BlockManagerTestUtil.UpdateState(nn1.GetNamesystem().GetBlockManager()); NUnit.Framework.Assert.AreEqual(0, nn0.GetNamesystem().GetUnderReplicatedBlocks() ); NUnit.Framework.Assert.AreEqual(0, nn1.GetNamesystem().GetUnderReplicatedBlocks() ); locs = nn1.GetRpcServer().GetBlockLocations(TestFile, 0, 1); NUnit.Framework.Assert.AreEqual("Standby should have registered that the block has replicas again" , 1, locs.Get(0).GetLocations().Length); } finally { cluster.Shutdown(); } }
public virtual void HSyncEndBlock_00() { int preferredBlockSize = 1024; Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, preferredBlockSize); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); DistributedFileSystem fileSystem = cluster.GetFileSystem(); FSDataOutputStream stm = null; try { Path path = new Path("/" + fName); stm = fileSystem.Create(path, true, 4096, (short)2, AppendTestUtil.BlockSize); System.Console.Out.WriteLine("Created file " + path.ToString()); ((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .EndBlock)); long currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual(0L, currentFileLength); LocatedBlocks blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0); NUnit.Framework.Assert.AreEqual(0, blocks.GetLocatedBlocks().Count); // write a block and call hsync(end_block) at the block boundary stm.Write(new byte[preferredBlockSize]); ((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .EndBlock)); currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual(preferredBlockSize, currentFileLength); blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0); NUnit.Framework.Assert.AreEqual(1, blocks.GetLocatedBlocks().Count); // call hsync then call hsync(end_block) immediately stm.Write(new byte[preferredBlockSize / 2]); stm.Hsync(); ((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .EndBlock)); currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual(preferredBlockSize + preferredBlockSize / 2, currentFileLength ); blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0); NUnit.Framework.Assert.AreEqual(2, blocks.GetLocatedBlocks().Count); stm.Write(new byte[preferredBlockSize / 4]); stm.Hsync(); currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual(preferredBlockSize + preferredBlockSize / 2 + preferredBlockSize / 4, currentFileLength); blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0); NUnit.Framework.Assert.AreEqual(3, blocks.GetLocatedBlocks().Count); } finally { IOUtils.Cleanup(null, stm, fileSystem); if (cluster != null) { cluster.Shutdown(); } } }
/// <summary> /// Verify the number of corrupted block replicas by fetching the block /// location from name node. /// </summary> /// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/> /// <exception cref="System.IO.FileNotFoundException"/> /// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/> /// <exception cref="System.IO.IOException"/> private void VerifyCorruptedBlockCount(Path filePath, int expectedReplicas) { LocatedBlocks lBlocks = dfs.dfs.GetNamenode().GetBlockLocations(filePath.ToUri(). GetPath(), 0, long.MaxValue); // we expect only the first block of the file is used for this test LocatedBlock firstLocatedBlock = lBlocks.Get(0); NUnit.Framework.Assert.AreEqual(expectedReplicas, firstLocatedBlock.GetLocations( ).Length); }
public virtual void TestReplicationError() { // create a file of replication factor of 1 Path fileName = new Path("/test.txt"); int fileLen = 1; DFSTestUtil.CreateFile(fs, fileName, 1, (short)1, 1L); DFSTestUtil.WaitReplication(fs, fileName, (short)1); // get the block belonged to the created file LocatedBlocks blocks = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), fileName .ToString(), 0, (long)fileLen); NUnit.Framework.Assert.AreEqual("Should only find 1 block", blocks.LocatedBlockCount (), 1); LocatedBlock block = blocks.Get(0); // bring up a second datanode cluster.StartDataNodes(conf, 1, true, null, null); cluster.WaitActive(); int sndNode = 1; DataNode datanode = cluster.GetDataNodes()[sndNode]; // replicate the block to the second datanode IPEndPoint target = datanode.GetXferAddress(); Socket s = Sharpen.Extensions.CreateSocket(target.Address, target.Port); // write the header. DataOutputStream @out = new DataOutputStream(s.GetOutputStream()); DataChecksum checksum = DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, 512 ); new Sender(@out).WriteBlock(block.GetBlock(), StorageType.Default, BlockTokenSecretManager .DummyToken, string.Empty, new DatanodeInfo[0], new StorageType[0], null, BlockConstructionStage .PipelineSetupCreate, 1, 0L, 0L, 0L, checksum, CachingStrategy.NewDefaultStrategy (), false, false, null); @out.Flush(); // close the connection before sending the content of the block @out.Close(); // the temporary block & meta files should be deleted string bpid = cluster.GetNamesystem().GetBlockPoolId(); FilePath storageDir = cluster.GetInstanceStorageDir(sndNode, 0); FilePath dir1 = MiniDFSCluster.GetRbwDir(storageDir, bpid); storageDir = cluster.GetInstanceStorageDir(sndNode, 1); FilePath dir2 = MiniDFSCluster.GetRbwDir(storageDir, bpid); while (dir1.ListFiles().Length != 0 || dir2.ListFiles().Length != 0) { Sharpen.Thread.Sleep(100); } // then increase the file's replication factor fs.SetReplication(fileName, (short)2); // replication should succeed DFSTestUtil.WaitReplication(fs, fileName, (short)1); // clean up the file fs.Delete(fileName, false); }
/// <exception cref="System.IO.IOException"/> public static LocatedBlock GetLastLocatedBlock(ClientProtocol namenode, string src ) { //get block info for the last block LocatedBlocks locations = namenode.GetBlockLocations(src, 0, long.MaxValue); IList <LocatedBlock> blocks = locations.GetLocatedBlocks(); DataNode.Log.Info("blocks.size()=" + blocks.Count); NUnit.Framework.Assert.IsTrue(blocks.Count > 0); return(blocks[blocks.Count - 1]); }
/// <summary>Select a datanode to service this request.</summary> /// <remarks> /// Select a datanode to service this request. /// Currently, this looks at no more than the first five blocks of a file, /// selecting a datanode randomly from the most represented. /// </remarks> /// <param name="conf"></param> /// <exception cref="System.IO.IOException"/> private DatanodeID PickSrcDatanode(LocatedBlocks blks, HdfsFileStatus i, Configuration conf) { if (i.GetLen() == 0 || blks.GetLocatedBlocks().Count <= 0) { // pick a random datanode NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext()); return(NamenodeJspHelper.GetRandomDatanode(nn)); } return(JspHelper.BestNode(blks, conf)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="Sharpen.TimeoutException"/> private void ChangeBlockLen(MiniDFSCluster cluster, int lenDelta) { Path fileName = new Path("/file1"); short ReplicationFactor = (short)1; FileSystem fs = cluster.GetFileSystem(); int fileLen = fs.GetConf().GetInt(DFSConfigKeys.DfsBytesPerChecksumKey, 512); DFSTestUtil.CreateFile(fs, fileName, fileLen, ReplicationFactor, 0); DFSTestUtil.WaitReplication(fs, fileName, ReplicationFactor); ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName); // Change the length of a replica for (int i = 0; i < cluster.GetDataNodes().Count; i++) { if (DFSTestUtil.ChangeReplicaLength(cluster, block, i, lenDelta)) { break; } } // increase the file's replication factor fs.SetReplication(fileName, (short)(ReplicationFactor + 1)); // block replication triggers corrupt block detection DFSClient dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort ()), fs.GetConf()); LocatedBlocks blocks = dfsClient.GetNamenode().GetBlockLocations(fileName.ToString (), 0, fileLen); if (lenDelta < 0) { // replica truncated while (!blocks.Get(0).IsCorrupt() || ReplicationFactor != blocks.Get(0).GetLocations ().Length) { Sharpen.Thread.Sleep(100); blocks = dfsClient.GetNamenode().GetBlockLocations(fileName.ToString(), 0, fileLen ); } } else { // no corruption detected; block replicated while (ReplicationFactor + 1 != blocks.Get(0).GetLocations().Length) { Sharpen.Thread.Sleep(100); blocks = dfsClient.GetNamenode().GetBlockLocations(fileName.ToString(), 0, fileLen ); } } fs.Delete(fileName, true); }
/// <exception cref="System.Exception"/> private void TestBadBlockReportOnTransfer(bool corruptBlockByDeletingBlockFile) { Configuration conf = new HdfsConfiguration(); FileSystem fs = null; DFSClient dfsClient = null; LocatedBlocks blocks = null; int replicaCount = 0; short replFactor = 1; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()), conf); // Create file with replication factor of 1 Path file1 = new Path("/tmp/testBadBlockReportOnTransfer/file1"); DFSTestUtil.CreateFile(fs, file1, 1024, replFactor, 0); DFSTestUtil.WaitReplication(fs, file1, replFactor); // Corrupt the block belonging to the created file ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, file1); int blockFilesCorrupted = corruptBlockByDeletingBlockFile ? cluster.CorruptBlockOnDataNodesByDeletingBlockFile (block) : cluster.CorruptBlockOnDataNodes(block); NUnit.Framework.Assert.AreEqual("Corrupted too few blocks", replFactor, blockFilesCorrupted ); // Increase replication factor, this should invoke transfer request // Receiving datanode fails on checksum and reports it to namenode replFactor = 2; fs.SetReplication(file1, replFactor); // Now get block details and check if the block is corrupt blocks = dfsClient.GetNamenode().GetBlockLocations(file1.ToString(), 0, long.MaxValue ); while (blocks.Get(0).IsCorrupt() != true) { try { Log.Info("Waiting until block is marked as corrupt..."); Sharpen.Thread.Sleep(1000); } catch (Exception) { } blocks = dfsClient.GetNamenode().GetBlockLocations(file1.ToString(), 0, long.MaxValue ); } replicaCount = blocks.Get(0).GetLocations().Length; NUnit.Framework.Assert.IsTrue(replicaCount == 1); cluster.Shutdown(); }
/// <exception cref="System.IO.IOException"/> private LocatedBlocks CreateFileGetBlocks(string filenamePrefix) { Path filePath = new Path("/" + filenamePrefix + ".dat"); // Write out a file with a few blocks, get block locations. DFSTestUtil.CreateFile(fs, filePath, BlockSize, BlockSize * NumBlocks, BlockSize, NumDatanodes, seed); // Get the block list for the file with the block locations. LocatedBlocks blocks = client.GetLocatedBlocks(filePath.ToString(), 0, BlockSize * NumBlocks); Assert.AssertThat(cluster.GetNamesystem().GetUnderReplicatedBlocks(), IS.Is(0L)); return(blocks); }
/// <summary>Test assumes that the file has a single block</summary> /// <exception cref="System.IO.IOException"/> private FilePath GetBlockForFile(Path path, bool exists) { LocatedBlocks blocks = nn.GetRpcServer().GetBlockLocations(path.ToString(), 0, long.MaxValue ); NUnit.Framework.Assert.AreEqual("The test helper functions assume that each file has a single block" , 1, blocks.GetLocatedBlocks().Count); ExtendedBlock block = blocks.GetLocatedBlocks()[0].GetBlock(); BlockLocalPathInfo bInfo = dn0.GetFSDataset().GetBlockLocalPathInfo(block); FilePath blockFile = new FilePath(bInfo.GetBlockPath()); NUnit.Framework.Assert.AreEqual(exists, blockFile.Exists()); return(blockFile); }
/// <summary>TC11: Racing rename</summary> /// <exception cref="System.Exception"/> private void TestTC11(bool appendToNewBlock) { Path p = new Path("/TC11/foo" + (appendToNewBlock ? "0" : "1")); System.Console.Out.WriteLine("p=" + p); //a. Create file and write one block of data. Close file. int len1 = (int)BlockSize; { FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } //b. Reopen file in "append" mode. Append half block of data. FSDataOutputStream out_1 = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag. Append, CreateFlag.NewBlock), 4096, null) : fs.Append(p); int len2 = (int)BlockSize / 2; AppendTestUtil.Write(out_1, len1, len2); out_1.Hflush(); //c. Rename file to file.new. Path pnew = new Path(p + ".new"); NUnit.Framework.Assert.IsTrue(fs.Rename(p, pnew)); //d. Close file handle that was opened in (b). out_1.Close(); //check block sizes long len = fs.GetFileStatus(pnew).GetLen(); LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(pnew.ToString (), 0L, len); int numblock = locatedblocks.LocatedBlockCount(); for (int i = 0; i < numblock; i++) { LocatedBlock lb = locatedblocks.Get(i); ExtendedBlock blk = lb.GetBlock(); long size = lb.GetBlockSize(); if (i < numblock - 1) { NUnit.Framework.Assert.AreEqual(BlockSize, size); } foreach (DatanodeInfo datanodeinfo in lb.GetLocations()) { DataNode dn = cluster.GetDataNode(datanodeinfo.GetIpcPort()); Block metainfo = DataNodeTestUtils.GetFSDataset(dn).GetStoredBlock(blk.GetBlockPoolId (), blk.GetBlockId()); NUnit.Framework.Assert.AreEqual(size, metainfo.GetNumBytes()); } } }
/// <summary>TC7: Corrupted replicas are present.</summary> /// <exception cref="System.IO.IOException">an exception might be thrown</exception> /// <exception cref="System.Exception"/> private void TestTC7(bool appendToNewBlock) { short repl = 2; Path p = new Path("/TC7/foo" + (appendToNewBlock ? "0" : "1")); System.Console.Out.WriteLine("p=" + p); //a. Create file with replication factor of 2. Write half block of data. Close file. int len1 = (int)(BlockSize / 2); { FSDataOutputStream @out = fs.Create(p, false, buffersize, repl, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } DFSTestUtil.WaitReplication(fs, p, repl); //b. Log into one datanode that has one replica of this block. // Find the block file on this datanode and truncate it to zero size. LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(p.ToString() , 0L, len1); NUnit.Framework.Assert.AreEqual(1, locatedblocks.LocatedBlockCount()); LocatedBlock lb = locatedblocks.Get(0); ExtendedBlock blk = lb.GetBlock(); NUnit.Framework.Assert.AreEqual(len1, lb.GetBlockSize()); DatanodeInfo[] datanodeinfos = lb.GetLocations(); NUnit.Framework.Assert.AreEqual(repl, datanodeinfos.Length); DataNode dn = cluster.GetDataNode(datanodeinfos[0].GetIpcPort()); FilePath f = DataNodeTestUtils.GetBlockFile(dn, blk.GetBlockPoolId(), blk.GetLocalBlock ()); RandomAccessFile raf = new RandomAccessFile(f, "rw"); AppendTestUtil.Log.Info("dn=" + dn + ", blk=" + blk + " (length=" + blk.GetNumBytes () + ")"); NUnit.Framework.Assert.AreEqual(len1, raf.Length()); raf.SetLength(0); raf.Close(); //c. Open file in "append mode". Append a new block worth of data. Close file. int len2 = (int)BlockSize; { FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append , CreateFlag.NewBlock), 4096, null) : fs.Append(p); AppendTestUtil.Write(@out, len1, len2); @out.Close(); } //d. Reopen file and read two blocks worth of data. AppendTestUtil.Check(fs, p, len1 + len2); }
/// <summary>Convert LocatedBlocks to a Json string.</summary> /// <exception cref="System.IO.IOException"/> public static string ToJsonString(LocatedBlocks locatedblocks) { if (locatedblocks == null) { return(null); } IDictionary <string, object> m = new SortedDictionary <string, object>(); m["fileLength"] = locatedblocks.GetFileLength(); m["isUnderConstruction"] = locatedblocks.IsUnderConstruction(); m["locatedBlocks"] = ToJsonArray(locatedblocks.GetLocatedBlocks()); m["lastLocatedBlock"] = ToJsonMap(locatedblocks.GetLastLocatedBlock()); m["isLastBlockComplete"] = locatedblocks.IsLastBlockComplete(); return(ToJsonString(typeof(LocatedBlocks), m)); }
public virtual void TestAppend2AfterSoftLimit() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1); //Set small soft-limit for lease long softLimit = 1L; long hardLimit = 9999999L; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.SetLeasePeriod(softLimit, hardLimit); cluster.WaitActive(); DistributedFileSystem fs = cluster.GetFileSystem(); DistributedFileSystem fs2 = new DistributedFileSystem(); fs2.Initialize(fs.GetUri(), conf); Path testPath = new Path("/testAppendAfterSoftLimit"); byte[] fileContents = AppendTestUtil.InitBuffer(32); // create a new file without closing FSDataOutputStream @out = fs.Create(testPath); @out.Write(fileContents); //Wait for > soft-limit Sharpen.Thread.Sleep(250); try { FSDataOutputStream appendStream2 = fs2.Append(testPath, EnumSet.Of(CreateFlag.Append , CreateFlag.NewBlock), 4096, null); appendStream2.Write(fileContents); appendStream2.Close(); NUnit.Framework.Assert.AreEqual(fileContents.Length, fs.GetFileStatus(testPath).GetLen ()); // make sure we now have 1 block since the first writer was revoked LocatedBlocks blks = fs.GetClient().GetLocatedBlocks(testPath.ToString(), 0L); NUnit.Framework.Assert.AreEqual(1, blks.GetLocatedBlocks().Count); foreach (LocatedBlock blk in blks.GetLocatedBlocks()) { NUnit.Framework.Assert.AreEqual(fileContents.Length, blk.GetBlockSize()); } } finally { fs.Close(); fs2.Close(); cluster.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> protected internal LocatedBlocks EnsureFileReplicasOnStorageType(Path path, StorageType storageType) { // Ensure that returned block locations returned are correct! Log.Info("Ensure path: " + path + " is on StorageType: " + storageType); Assert.AssertThat(fs.Exists(path), IS.Is(true)); long fileLength = client.GetFileInfo(path.ToString()).GetLen(); LocatedBlocks locatedBlocks = client.GetLocatedBlocks(path.ToString(), 0, fileLength ); foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks()) { Assert.AssertThat(locatedBlock.GetStorageTypes()[0], IS.Is(storageType)); } return(locatedBlocks); }
public virtual void TestBlockRecoveryWithLessMetafile() { Configuration conf = new Configuration(); conf.Set(DFSConfigKeys.DfsBlockLocalPathAccessUserKey, UserGroupInformation.GetCurrentUser ().GetShortUserName()); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); Path file = new Path("/testRecoveryFile"); DistributedFileSystem dfs = cluster.GetFileSystem(); FSDataOutputStream @out = dfs.Create(file); int count = 0; while (count < 2 * 1024 * 1024) { @out.WriteBytes("Data"); count += 4; } @out.Hsync(); // abort the original stream ((DFSOutputStream)@out.GetWrappedStream()).Abort(); LocatedBlocks locations = cluster.GetNameNodeRpc().GetBlockLocations(file.ToString (), 0, count); ExtendedBlock block = locations.Get(0).GetBlock(); DataNode dn = cluster.GetDataNodes()[0]; BlockLocalPathInfo localPathInfo = dn.GetBlockLocalPathInfo(block, null); FilePath metafile = new FilePath(localPathInfo.GetMetaPath()); NUnit.Framework.Assert.IsTrue(metafile.Exists()); // reduce the block meta file size RandomAccessFile raf = new RandomAccessFile(metafile, "rw"); raf.SetLength(metafile.Length() - 20); raf.Close(); // restart DN to make replica to RWR MiniDFSCluster.DataNodeProperties dnProp = cluster.StopDataNode(0); cluster.RestartDataNode(dnProp, true); // try to recover the lease DistributedFileSystem newdfs = (DistributedFileSystem)FileSystem.NewInstance(cluster .GetConfiguration(0)); count = 0; while (++count < 10 && !newdfs.RecoverLease(file)) { Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsTrue("File should be closed", newdfs.RecoverLease(file)); }