/// <summary>TC12: Append to partial CRC chunk</summary> /// <exception cref="System.Exception"/> private void TestTC12(bool appendToNewBlock) { Path p = new Path("/TC12/foo" + (appendToNewBlock ? "0" : "1")); System.Console.Out.WriteLine("p=" + p); //a. Create file with a block size of 64KB // and a default io.bytes.per.checksum of 512 bytes. // Write 25687 bytes of data. Close file. int len1 = 25687; { FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } //b. Reopen file in "append" mode. Append another 5877 bytes of data. Close file. int len2 = 5877; { FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append , CreateFlag.NewBlock), 4096, null) : fs.Append(p); AppendTestUtil.Write(@out, len1, len2); @out.Close(); } //c. Reopen file and read 25687+5877 bytes of data from file. Close file. AppendTestUtil.Check(fs, p, len1 + len2); if (appendToNewBlock) { LocatedBlocks blks = fs.dfs.GetLocatedBlocks(p.ToString(), 0); NUnit.Framework.Assert.AreEqual(2, blks.GetLocatedBlocks().Count); NUnit.Framework.Assert.AreEqual(len1, blks.GetLocatedBlocks()[0].GetBlockSize()); NUnit.Framework.Assert.AreEqual(len2, blks.GetLocatedBlocks()[1].GetBlockSize()); AppendTestUtil.Check(fs, p, 0, len1); AppendTestUtil.Check(fs, p, len1, len2); } }
/// <exception cref="System.IO.IOException"/> private LocatedBlock GetLocatedBlock() { LocatedBlocks locatedBlocks = client.GetLocatedBlocks(Path.ToString(), 0, BlockSize ); Assert.AssertThat(locatedBlocks.GetLocatedBlocks().Count, CoreMatchers.Is(1)); return(Iterables.GetOnlyElement(locatedBlocks.GetLocatedBlocks())); }
public virtual void HSyncEndBlock_00() { int preferredBlockSize = 1024; Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, preferredBlockSize); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); DistributedFileSystem fileSystem = cluster.GetFileSystem(); FSDataOutputStream stm = null; try { Path path = new Path("/" + fName); stm = fileSystem.Create(path, true, 4096, (short)2, AppendTestUtil.BlockSize); System.Console.Out.WriteLine("Created file " + path.ToString()); ((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .EndBlock)); long currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual(0L, currentFileLength); LocatedBlocks blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0); NUnit.Framework.Assert.AreEqual(0, blocks.GetLocatedBlocks().Count); // write a block and call hsync(end_block) at the block boundary stm.Write(new byte[preferredBlockSize]); ((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .EndBlock)); currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual(preferredBlockSize, currentFileLength); blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0); NUnit.Framework.Assert.AreEqual(1, blocks.GetLocatedBlocks().Count); // call hsync then call hsync(end_block) immediately stm.Write(new byte[preferredBlockSize / 2]); stm.Hsync(); ((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .EndBlock)); currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual(preferredBlockSize + preferredBlockSize / 2, currentFileLength ); blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0); NUnit.Framework.Assert.AreEqual(2, blocks.GetLocatedBlocks().Count); stm.Write(new byte[preferredBlockSize / 4]); stm.Hsync(); currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual(preferredBlockSize + preferredBlockSize / 2 + preferredBlockSize / 4, currentFileLength); blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0); NUnit.Framework.Assert.AreEqual(3, blocks.GetLocatedBlocks().Count); } finally { IOUtils.Cleanup(null, stm, fileSystem); if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>Test assumes that the file has a single block</summary> /// <exception cref="System.IO.IOException"/> private FilePath GetBlockForFile(Path path, bool exists) { LocatedBlocks blocks = nn.GetRpcServer().GetBlockLocations(path.ToString(), 0, long.MaxValue ); NUnit.Framework.Assert.AreEqual("The test helper functions assume that each file has a single block" , 1, blocks.GetLocatedBlocks().Count); ExtendedBlock block = blocks.GetLocatedBlocks()[0].GetBlock(); BlockLocalPathInfo bInfo = dn0.GetFSDataset().GetBlockLocalPathInfo(block); FilePath blockFile = new FilePath(bInfo.GetBlockPath()); NUnit.Framework.Assert.AreEqual(exists, blockFile.Exists()); return(blockFile); }
public virtual void TestAppend2AfterSoftLimit() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1); //Set small soft-limit for lease long softLimit = 1L; long hardLimit = 9999999L; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.SetLeasePeriod(softLimit, hardLimit); cluster.WaitActive(); DistributedFileSystem fs = cluster.GetFileSystem(); DistributedFileSystem fs2 = new DistributedFileSystem(); fs2.Initialize(fs.GetUri(), conf); Path testPath = new Path("/testAppendAfterSoftLimit"); byte[] fileContents = AppendTestUtil.InitBuffer(32); // create a new file without closing FSDataOutputStream @out = fs.Create(testPath); @out.Write(fileContents); //Wait for > soft-limit Sharpen.Thread.Sleep(250); try { FSDataOutputStream appendStream2 = fs2.Append(testPath, EnumSet.Of(CreateFlag.Append , CreateFlag.NewBlock), 4096, null); appendStream2.Write(fileContents); appendStream2.Close(); NUnit.Framework.Assert.AreEqual(fileContents.Length, fs.GetFileStatus(testPath).GetLen ()); // make sure we now have 1 block since the first writer was revoked LocatedBlocks blks = fs.GetClient().GetLocatedBlocks(testPath.ToString(), 0L); NUnit.Framework.Assert.AreEqual(1, blks.GetLocatedBlocks().Count); foreach (LocatedBlock blk in blks.GetLocatedBlocks()) { NUnit.Framework.Assert.AreEqual(fileContents.Length, blk.GetBlockSize()); } } finally { fs.Close(); fs2.Close(); cluster.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> public static DatanodeInfo BestNode(LocatedBlocks blks, Configuration conf) { Dictionary <DatanodeInfo, JspHelper.NodeRecord> map = new Dictionary <DatanodeInfo, JspHelper.NodeRecord>(); foreach (LocatedBlock block in blks.GetLocatedBlocks()) { DatanodeInfo[] nodes = block.GetLocations(); foreach (DatanodeInfo node in nodes) { JspHelper.NodeRecord record = map[node]; if (record == null) { map[node] = new JspHelper.NodeRecord(node, 1); } else { record.frequency++; } } } JspHelper.NodeRecord[] nodes_1 = Sharpen.Collections.ToArray(map.Values, new JspHelper.NodeRecord [map.Count]); Arrays.Sort(nodes_1, new JspHelper.NodeRecordComparator()); return(BestNode(nodes_1, false)); }
public virtual void TestFallbackToDiskPartial() { StartUpCluster(true, 2); string MethodName = GenericTestUtils.GetMethodName(); Path path = new Path("/" + MethodName + ".dat"); MakeTestFile(path, BlockSize * 5, true); // Sleep for a short time to allow the lazy writer thread to do its job Sharpen.Thread.Sleep(6 * LazyWriterIntervalSec * 1000); TriggerBlockReport(); int numBlocksOnRamDisk = 0; int numBlocksOnDisk = 0; long fileLength = client.GetFileInfo(path.ToString()).GetLen(); LocatedBlocks locatedBlocks = client.GetLocatedBlocks(path.ToString(), 0, fileLength ); foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks()) { if (locatedBlock.GetStorageTypes()[0] == StorageType.RamDisk) { numBlocksOnRamDisk++; } else { if (locatedBlock.GetStorageTypes()[0] == StorageType.Default) { numBlocksOnDisk++; } } } // Since eviction is asynchronous, depending on the timing of eviction // wrt writes, we may get 2 or less blocks on RAM disk. System.Diagnostics.Debug.Assert((numBlocksOnRamDisk <= 2)); System.Diagnostics.Debug.Assert((numBlocksOnDisk >= 3)); }
public virtual void TestGetBlockLocations() { NamenodeProtocols namenode = cluster.GetNameNodeRpc(); Path p = new Path(BaseDir, "file2.dat"); string src = p.ToString(); FSDataOutputStream @out = TestFileCreation.CreateFile(hdfs, p, 3); // write a half block int len = (int)(((uint)BlockSize) >> 1); WriteFile(p, @out, len); for (int i = 1; i < NumBlocks;) { // verify consistency LocatedBlocks lb = namenode.GetBlockLocations(src, 0, len); IList <LocatedBlock> blocks = lb.GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(i, blocks.Count); Block b = blocks[blocks.Count - 1].GetBlock().GetLocalBlock(); NUnit.Framework.Assert.IsTrue(b is BlockInfoContiguousUnderConstruction); if (++i < NumBlocks) { // write one more block WriteFile(p, @out, BlockSize); len += BlockSize; } } // close file @out.Close(); }
public virtual void TestBlockHasMultipleReplicasOnSameDN() { string filename = MakeFileName(GenericTestUtils.GetMethodName()); Path filePath = new Path(filename); // Write out a file with a few blocks. DFSTestUtil.CreateFile(fs, filePath, BlockSize, BlockSize * NumBlocks, BlockSize, NumDatanodes, seed); // Get the block list for the file with the block locations. LocatedBlocks locatedBlocks = client.GetLocatedBlocks(filePath.ToString(), 0, BlockSize * NumBlocks); // Generate a fake block report from one of the DataNodes, such // that it reports one copy of each block on either storage. DataNode dn = cluster.GetDataNodes()[0]; DatanodeRegistration dnReg = dn.GetDNRegistrationForBP(bpid); StorageBlockReport[] reports = new StorageBlockReport[cluster.GetStoragesPerDatanode ()]; AList <Replica> blocks = new AList <Replica>(); foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks()) { Block localBlock = locatedBlock.GetBlock().GetLocalBlock(); blocks.AddItem(new FinalizedReplica(localBlock, null, null)); } BlockListAsLongs bll = BlockListAsLongs.Encode(blocks); for (int i = 0; i < cluster.GetStoragesPerDatanode(); ++i) { FsVolumeSpi v = dn.GetFSDataset().GetVolumes()[i]; DatanodeStorage dns = new DatanodeStorage(v.GetStorageID()); reports[i] = new StorageBlockReport(dns, bll); } // Should not assert! cluster.GetNameNodeRpc().BlockReport(dnReg, bpid, reports, new BlockReportContext (1, 0, Runtime.NanoTime())); // Get the block locations once again. locatedBlocks = client.GetLocatedBlocks(filename, 0, BlockSize * NumBlocks); // Make sure that each block has two replicas, one on each DataNode. foreach (LocatedBlock locatedBlock_1 in locatedBlocks.GetLocatedBlocks()) { DatanodeInfo[] locations = locatedBlock_1.GetLocations(); Assert.AssertThat(locations.Length, IS.Is((int)NumDatanodes)); Assert.AssertThat(locations[0].GetDatanodeUuid(), CoreMatchers.Not(locations[1].GetDatanodeUuid ())); } }
/// <summary> /// Create a cluster with even distribution, and a new empty node is added to /// the cluster, then test rack locality for balancer policy. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestBalancerWithRackLocality() { Configuration conf = CreateConf(); long[] capacities = new long[] { Capacity, Capacity }; string[] racks = new string[] { Rack0, Rack1 }; string[] nodeGroups = new string[] { Nodegroup0, Nodegroup1 }; int numOfDatanodes = capacities.Length; NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities .Length).Racks(racks).SimulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder); try { cluster.WaitActive(); client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem( 0).GetUri()).GetProxy(); long totalCapacity = TestBalancer.Sum(capacities); // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity * 3 / 10; long length = totalUsedSpace / numOfDatanodes; TestBalancer.CreateFile(cluster, filePath, length, (short)numOfDatanodes, 0); LocatedBlocks lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length ); ICollection <ExtendedBlock> before = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0 ); long newCapacity = Capacity; string newRack = Rack1; string newNodeGroup = Nodegroup2; // start up an empty node with the same capacity and on the same rack cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[] { newCapacity }, new string[] { newNodeGroup }); totalCapacity += newCapacity; // run balancer and validate results RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity); lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length); ICollection <ExtendedBlock> after = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0); NUnit.Framework.Assert.AreEqual(before, after); } finally { cluster.Shutdown(); } }
public virtual void TestRetryAddBlockWhileInChooseTarget() { string src = "/testRetryAddBlockWhileInChooseTarget"; FSNamesystem ns = cluster.GetNamesystem(); NamenodeProtocols nn = cluster.GetNameNodeRpc(); // create file nn.Create(src, FsPermission.GetFileDefault(), "clientName", new EnumSetWritable <CreateFlag >(EnumSet.Of(CreateFlag.Create)), true, (short)3, 1024, null); // start first addBlock() Log.Info("Starting first addBlock for " + src); LocatedBlock[] onRetryBlock = new LocatedBlock[1]; DatanodeStorageInfo[] targets = ns.GetNewBlockTargets(src, INodeId.GrandfatherInodeId , "clientName", null, null, null, onRetryBlock); NUnit.Framework.Assert.IsNotNull("Targets must be generated", targets); // run second addBlock() Log.Info("Starting second addBlock for " + src); nn.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId, null); NUnit.Framework.Assert.IsTrue("Penultimate block must be complete", CheckFileProgress (src, false)); LocatedBlocks lbs = nn.GetBlockLocations(src, 0, long.MaxValue); NUnit.Framework.Assert.AreEqual("Must be one block", 1, lbs.GetLocatedBlocks().Count ); LocatedBlock lb2 = lbs.Get(0); NUnit.Framework.Assert.AreEqual("Wrong replication", Replication, lb2.GetLocations ().Length); // continue first addBlock() LocatedBlock newBlock = ns.StoreAllocatedBlock(src, INodeId.GrandfatherInodeId, "clientName" , null, targets); NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb2.GetBlock(), newBlock. GetBlock()); // check locations lbs = nn.GetBlockLocations(src, 0, long.MaxValue); NUnit.Framework.Assert.AreEqual("Must be one block", 1, lbs.GetLocatedBlocks().Count ); LocatedBlock lb1 = lbs.Get(0); NUnit.Framework.Assert.AreEqual("Wrong replication", Replication, lb1.GetLocations ().Length); NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb1.GetBlock(), lb2.GetBlock ()); }
/// <summary> /// Check the blocks of dst file are cleaned after rename with overwrite /// Restart NN to check the rename successfully /// </summary> /// <exception cref="System.Exception"/> public virtual void TestRenameWithOverwrite() { short replFactor = 2; long blockSize = 512; Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(replFactor ).Build(); DistributedFileSystem dfs = cluster.GetFileSystem(); try { long fileLen = blockSize * 3; string src = "/foo/src"; string dst = "/foo/dst"; Path srcPath = new Path(src); Path dstPath = new Path(dst); DFSTestUtil.CreateFile(dfs, srcPath, fileLen, replFactor, 1); DFSTestUtil.CreateFile(dfs, dstPath, fileLen, replFactor, 1); LocatedBlocks lbs = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), dst, 0, fileLen); BlockManager bm = NameNodeAdapter.GetNamesystem(cluster.GetNameNode()).GetBlockManager (); NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(lbs.GetLocatedBlocks()[0].GetBlock ().GetLocalBlock()) != null); dfs.Rename(srcPath, dstPath, Options.Rename.Overwrite); NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(lbs.GetLocatedBlocks()[0].GetBlock ().GetLocalBlock()) == null); // Restart NN and check the rename successfully cluster.RestartNameNodes(); NUnit.Framework.Assert.IsFalse(dfs.Exists(srcPath)); NUnit.Framework.Assert.IsTrue(dfs.Exists(dstPath)); } finally { if (dfs != null) { dfs.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>Select a datanode to service this request.</summary> /// <remarks> /// Select a datanode to service this request. /// Currently, this looks at no more than the first five blocks of a file, /// selecting a datanode randomly from the most represented. /// </remarks> /// <param name="conf"></param> /// <exception cref="System.IO.IOException"/> private DatanodeID PickSrcDatanode(LocatedBlocks blks, HdfsFileStatus i, Configuration conf) { if (i.GetLen() == 0 || blks.GetLocatedBlocks().Count <= 0) { // pick a random datanode NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext()); return(NamenodeJspHelper.GetRandomDatanode(nn)); } return(JspHelper.BestNode(blks, conf)); }
/// <exception cref="System.IO.IOException"/> public static LocatedBlock GetLastLocatedBlock(ClientProtocol namenode, string src ) { //get block info for the last block LocatedBlocks locations = namenode.GetBlockLocations(src, 0, long.MaxValue); IList <LocatedBlock> blocks = locations.GetLocatedBlocks(); DataNode.Log.Info("blocks.size()=" + blocks.Count); NUnit.Framework.Assert.IsTrue(blocks.Count > 0); return(blocks[blocks.Count - 1]); }
/// <summary>Make sure at least one non-transient volume has a saved copy of the replica. /// </summary> /// <remarks> /// Make sure at least one non-transient volume has a saved copy of the replica. /// An infinite loop is used to ensure the async lazy persist tasks are completely /// done before verification. Caller of ensureLazyPersistBlocksAreSaved expects /// either a successful pass or timeout failure. /// </remarks> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected internal void EnsureLazyPersistBlocksAreSaved(LocatedBlocks locatedBlocks ) { string bpid = cluster.GetNamesystem().GetBlockPoolId(); IList <FsVolumeSpi> volumes = cluster.GetDataNodes()[0].GetFSDataset().GetVolumes( ); ICollection <long> persistedBlockIds = new HashSet <long>(); while (persistedBlockIds.Count < locatedBlocks.GetLocatedBlocks().Count) { // Take 1 second sleep before each verification iteration Sharpen.Thread.Sleep(1000); foreach (LocatedBlock lb in locatedBlocks.GetLocatedBlocks()) { foreach (FsVolumeSpi v in volumes) { if (v.IsTransientStorage()) { continue; } FsVolumeImpl volume = (FsVolumeImpl)v; FilePath lazyPersistDir = volume.GetBlockPoolSlice(bpid).GetLazypersistDir(); long blockId = lb.GetBlock().GetBlockId(); FilePath targetDir = DatanodeUtil.IdToBlockDir(lazyPersistDir, blockId); FilePath blockFile = new FilePath(targetDir, lb.GetBlock().GetBlockName()); if (blockFile.Exists()) { // Found a persisted copy for this block and added to the Set persistedBlockIds.AddItem(blockId); } } } } // We should have found a persisted copy for each located block. Assert.AssertThat(persistedBlockIds.Count, IS.Is(locatedBlocks.GetLocatedBlocks() .Count)); }
/// <summary>Convert LocatedBlocks to a Json string.</summary> /// <exception cref="System.IO.IOException"/> public static string ToJsonString(LocatedBlocks locatedblocks) { if (locatedblocks == null) { return(null); } IDictionary <string, object> m = new SortedDictionary <string, object>(); m["fileLength"] = locatedblocks.GetFileLength(); m["isUnderConstruction"] = locatedblocks.IsUnderConstruction(); m["locatedBlocks"] = ToJsonArray(locatedblocks.GetLocatedBlocks()); m["lastLocatedBlock"] = ToJsonMap(locatedblocks.GetLastLocatedBlock()); m["isLastBlockComplete"] = locatedblocks.IsLastBlockComplete(); return(ToJsonString(typeof(LocatedBlocks), m)); }
/// <exception cref="System.IO.IOException"/> protected internal LocatedBlocks EnsureFileReplicasOnStorageType(Path path, StorageType storageType) { // Ensure that returned block locations returned are correct! Log.Info("Ensure path: " + path + " is on StorageType: " + storageType); Assert.AssertThat(fs.Exists(path), IS.Is(true)); long fileLength = client.GetFileInfo(path.ToString()).GetLen(); LocatedBlocks locatedBlocks = client.GetLocatedBlocks(path.ToString(), 0, fileLength ); foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks()) { Assert.AssertThat(locatedBlock.GetStorageTypes()[0], IS.Is(storageType)); } return(locatedBlocks); }
/// <exception cref="System.IO.IOException"/> public virtual void VerifyIncrementalBlockReports(bool splitReports) { // Get the block list for the file with the block locations. LocatedBlocks blocks = CreateFileGetBlocks(GenericTestUtils.GetMethodName()); // We will send 'fake' incremental block reports to the NN that look // like they originated from DN 0. StorageReceivedDeletedBlocks[] reports = new StorageReceivedDeletedBlocks[dn0.GetFSDataset ().GetVolumes().Count]; // Lie to the NN that one block on each storage has been deleted. for (int i = 0; i < reports.Length; ++i) { FsVolumeSpi volume = dn0.GetFSDataset().GetVolumes()[i]; bool foundBlockOnStorage = false; ReceivedDeletedBlockInfo[] rdbi = new ReceivedDeletedBlockInfo[1]; // Find the first block on this storage and mark it as deleted for the // report. foreach (LocatedBlock block in blocks.GetLocatedBlocks()) { if (block.GetStorageIDs()[0].Equals(volume.GetStorageID())) { rdbi[0] = new ReceivedDeletedBlockInfo(block.GetBlock().GetLocalBlock(), ReceivedDeletedBlockInfo.BlockStatus .DeletedBlock, null); foundBlockOnStorage = true; break; } } NUnit.Framework.Assert.IsTrue(foundBlockOnStorage); reports[i] = new StorageReceivedDeletedBlocks(volume.GetStorageID(), rdbi); if (splitReports) { // If we are splitting reports then send the report for this storage now. StorageReceivedDeletedBlocks[] singletonReport = new StorageReceivedDeletedBlocks [] { reports[i] }; cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dn0Reg, poolId, singletonReport); } } if (!splitReports) { // Send a combined report. cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dn0Reg, poolId, reports); } // Make sure that the deleted block from each storage was picked up // by the NameNode. Assert.AssertThat(cluster.GetNamesystem().GetMissingBlocksCount(), IS.Is((long)reports .Length)); }
/// <returns>true if it is necessary to run another round of migration</returns> private bool ProcessFile(string fullPath, HdfsLocatedFileStatus status) { byte policyId = status.GetStoragePolicy(); // currently we ignore files with unspecified storage policy if (policyId == BlockStoragePolicySuite.IdUnspecified) { return(false); } BlockStoragePolicy policy = this._enclosing.blockStoragePolicies[policyId]; if (policy == null) { Org.Apache.Hadoop.Hdfs.Server.Mover.Mover.Log.Warn("Failed to get the storage policy of file " + fullPath); return(false); } IList <StorageType> types = policy.ChooseStorageTypes(status.GetReplication()); LocatedBlocks locatedBlocks = status.GetBlockLocations(); bool hasRemaining = false; bool lastBlkComplete = locatedBlocks.IsLastBlockComplete(); IList <LocatedBlock> lbs = locatedBlocks.GetLocatedBlocks(); for (int i = 0; i < lbs.Count; i++) { if (i == lbs.Count - 1 && !lastBlkComplete) { // last block is incomplete, skip it continue; } LocatedBlock lb = lbs[i]; Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes( )); if (!diff.RemoveOverlap(true)) { if (this.ScheduleMoves4Block(diff, lb)) { hasRemaining |= (diff.existing.Count > 1 && diff.expected.Count > 1); } } } return(hasRemaining); }
// Waits for all of the blocks to have expected replication /// <exception cref="System.IO.IOException"/> private void WaitForBlockReplication(string filename, ClientProtocol namenode, int expected, long maxWaitSec) { long start = Time.MonotonicNow(); //wait for all the blocks to be replicated; Log.Info("Checking for block replication for " + filename); while (true) { bool replOk = true; LocatedBlocks blocks = namenode.GetBlockLocations(filename, 0, long.MaxValue); for (IEnumerator <LocatedBlock> iter = blocks.GetLocatedBlocks().GetEnumerator(); iter.HasNext();) { LocatedBlock block = iter.Next(); int actual = block.GetLocations().Length; if (actual < expected) { Log.Info("Not enough replicas for " + block.GetBlock() + " yet. Expecting " + expected + ", got " + actual + "."); replOk = false; break; } } if (replOk) { return; } if (maxWaitSec > 0 && (Time.MonotonicNow() - start) > (maxWaitSec * 1000)) { throw new IOException("Timedout while waiting for all blocks to " + " be replicated for " + filename); } try { Sharpen.Thread.Sleep(500); } catch (Exception) { } } }
protected internal bool VerifyBlockDeletedFromDir(FilePath dir, LocatedBlocks locatedBlocks ) { foreach (LocatedBlock lb in locatedBlocks.GetLocatedBlocks()) { FilePath targetDir = DatanodeUtil.IdToBlockDir(dir, lb.GetBlock().GetBlockId()); FilePath blockFile = new FilePath(targetDir, lb.GetBlock().GetBlockName()); if (blockFile.Exists()) { Log.Warn("blockFile: " + blockFile.GetAbsolutePath() + " exists after deletion."); return(false); } FilePath metaFile = new FilePath(targetDir, DatanodeUtil.GetMetaName(lb.GetBlock( ).GetBlockName(), lb.GetBlock().GetGenerationStamp())); if (metaFile.Exists()) { Log.Warn("metaFile: " + metaFile.GetAbsolutePath() + " exists after deletion."); return(false); } } return(true); }
/// <summary> /// Verify that the DataNode sends a single incremental block report for all /// storages. /// </summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestDataNodeDoesNotSplitReports() { LocatedBlocks blocks = CreateFileGetBlocks(GenericTestUtils.GetMethodName()); Assert.AssertThat(cluster.GetDataNodes().Count, IS.Is(1)); // Remove all blocks from the DataNode. foreach (LocatedBlock block in blocks.GetLocatedBlocks()) { dn0.NotifyNamenodeDeletedBlock(block.GetBlock(), block.GetStorageIDs()[0]); } Log.Info("Triggering report after deleting blocks"); long ops = MetricsAsserts.GetLongCounter("BlockReceivedAndDeletedOps", MetricsAsserts.GetMetrics (NnMetrics)); // Trigger a report to the NameNode and give it a few seconds. DataNodeTestUtils.TriggerBlockReport(dn0); Sharpen.Thread.Sleep(5000); // Ensure that NameNodeRpcServer.blockReceivedAndDeletes is invoked // exactly once after we triggered the report. MetricsAsserts.AssertCounter("BlockReceivedAndDeletedOps", ops + 1, MetricsAsserts.GetMetrics (NnMetrics)); }
public virtual void TestGetBlockLocations() { Path root = new Path("/"); Path file = new Path("/file"); DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed); // take a snapshot on root SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1"); Path fileInSnapshot = SnapshotTestHelper.GetSnapshotPath(root, "s1", file.GetName ()); FileStatus status = hdfs.GetFileStatus(fileInSnapshot); // make sure we record the size for the file NUnit.Framework.Assert.AreEqual(Blocksize, status.GetLen()); // append data to file DFSTestUtil.AppendFile(hdfs, file, Blocksize - 1); status = hdfs.GetFileStatus(fileInSnapshot); // the size of snapshot file should still be BLOCKSIZE NUnit.Framework.Assert.AreEqual(Blocksize, status.GetLen()); // the size of the file should be (2 * BLOCKSIZE - 1) status = hdfs.GetFileStatus(file); NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, status.GetLen()); // call DFSClient#callGetBlockLocations for the file in snapshot LocatedBlocks blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc (), fileInSnapshot.ToString(), 0, long.MaxValue); IList <LocatedBlock> blockList = blocks.GetLocatedBlocks(); // should be only one block NUnit.Framework.Assert.AreEqual(Blocksize, blocks.GetFileLength()); NUnit.Framework.Assert.AreEqual(1, blockList.Count); // check the last block LocatedBlock lastBlock = blocks.GetLastLocatedBlock(); NUnit.Framework.Assert.AreEqual(0, lastBlock.GetStartOffset()); NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetBlockSize()); // take another snapshot SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2"); Path fileInSnapshot2 = SnapshotTestHelper.GetSnapshotPath(root, "s2", file.GetName ()); // append data to file without closing HdfsDataOutputStream @out = AppendFileWithoutClosing(file, Blocksize); @out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength)); status = hdfs.GetFileStatus(fileInSnapshot2); // the size of snapshot file should be BLOCKSIZE*2-1 NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, status.GetLen()); // the size of the file should be (3 * BLOCKSIZE - 1) status = hdfs.GetFileStatus(file); NUnit.Framework.Assert.AreEqual(Blocksize * 3 - 1, status.GetLen()); blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), fileInSnapshot2 .ToString(), 0, long.MaxValue); NUnit.Framework.Assert.IsFalse(blocks.IsUnderConstruction()); NUnit.Framework.Assert.IsTrue(blocks.IsLastBlockComplete()); blockList = blocks.GetLocatedBlocks(); // should be 2 blocks NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, blocks.GetFileLength()); NUnit.Framework.Assert.AreEqual(2, blockList.Count); // check the last block lastBlock = blocks.GetLastLocatedBlock(); NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetStartOffset()); NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetBlockSize()); blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), fileInSnapshot2 .ToString(), Blocksize, 0); blockList = blocks.GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(1, blockList.Count); // check blocks for file being written blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), file.ToString (), 0, long.MaxValue); blockList = blocks.GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(3, blockList.Count); NUnit.Framework.Assert.IsTrue(blocks.IsUnderConstruction()); NUnit.Framework.Assert.IsFalse(blocks.IsLastBlockComplete()); lastBlock = blocks.GetLastLocatedBlock(); NUnit.Framework.Assert.AreEqual(Blocksize * 2, lastBlock.GetStartOffset()); NUnit.Framework.Assert.AreEqual(Blocksize - 1, lastBlock.GetBlockSize()); @out.Close(); }
/// <summary> /// The method starts new cluster with defined Configuration; creates a file /// with specified block_size and writes 10 equal sections in it; it also calls /// hflush/hsync after each write and throws an IOException in case of an error. /// </summary> /// <param name="conf">cluster configuration</param> /// <param name="fileName">of the file to be created and processed as required</param> /// <param name="block_size">value to be used for the file's creation</param> /// <param name="replicas">is the number of replicas</param> /// <param name="isSync">hsync or hflush</param> /// <param name="syncFlags">specify the semantic of the sync/flush</param> /// <exception cref="System.IO.IOException">in case of any errors</exception> public static void DoTheJob(Configuration conf, string fileName, long block_size, short replicas, bool isSync, EnumSet <HdfsDataOutputStream.SyncFlag> syncFlags) { byte[] fileContent; int Sections = 10; fileContent = AppendTestUtil.InitBuffer(AppendTestUtil.FileSize); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(replicas). Build(); // Make sure we work with DFS in order to utilize all its functionality DistributedFileSystem fileSystem = cluster.GetFileSystem(); FSDataInputStream @is; try { Path path = new Path(fileName); string pathName = new Path(fileSystem.GetWorkingDirectory(), path).ToUri().GetPath (); FSDataOutputStream stm = fileSystem.Create(path, false, 4096, replicas, block_size ); System.Console.Out.WriteLine("Created file " + fileName); int tenth = AppendTestUtil.FileSize / Sections; int rounding = AppendTestUtil.FileSize - tenth * Sections; for (int i = 0; i < Sections; i++) { System.Console.Out.WriteLine("Writing " + (tenth * i) + " to " + (tenth * (i + 1) ) + " section to file " + fileName); // write to the file stm.Write(fileContent, tenth * i, tenth); // Wait while hflush/hsync pushes all packets through built pipeline if (isSync) { ((DFSOutputStream)stm.GetWrappedStream()).Hsync(syncFlags); } else { ((DFSOutputStream)stm.GetWrappedStream()).Hflush(); } // Check file length if updatelength is required if (isSync && syncFlags.Contains(HdfsDataOutputStream.SyncFlag.UpdateLength)) { long currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual("File size doesn't match for hsync/hflush with updating the length" , tenth * (i + 1), currentFileLength); } else { if (isSync && syncFlags.Contains(HdfsDataOutputStream.SyncFlag.EndBlock)) { LocatedBlocks blocks = fileSystem.dfs.GetLocatedBlocks(pathName, 0); NUnit.Framework.Assert.AreEqual(i + 1, blocks.GetLocatedBlocks().Count); } } byte[] toRead = new byte[tenth]; byte[] expected = new byte[tenth]; System.Array.Copy(fileContent, tenth * i, expected, 0, tenth); // Open the same file for read. Need to create new reader after every write operation(!) @is = fileSystem.Open(path); @is.Seek(tenth * i); int readBytes = @is.Read(toRead, 0, tenth); System.Console.Out.WriteLine("Has read " + readBytes); NUnit.Framework.Assert.IsTrue("Should've get more bytes", (readBytes > 0) && (readBytes <= tenth)); @is.Close(); CheckData(toRead, 0, readBytes, expected, "Partial verification"); } System.Console.Out.WriteLine("Writing " + (tenth * Sections) + " to " + (tenth * Sections + rounding) + " section to file " + fileName); stm.Write(fileContent, tenth * Sections, rounding); stm.Close(); NUnit.Framework.Assert.AreEqual("File size doesn't match ", AppendTestUtil.FileSize , fileSystem.GetFileStatus(path).GetLen()); AppendTestUtil.CheckFullFile(fileSystem, path, fileContent.Length, fileContent, "hflush()" ); } finally { fileSystem.Close(); cluster.Shutdown(); } }
public virtual void TestCopyOnWrite() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = cluster.GetFileSystem(); IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort()); DFSClient client = new DFSClient(addr, conf); try { // create a new file, write to it and close it. // Path file1 = new Path("/filestatus.dat"); FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1); WriteFile(stm); stm.Close(); // Get a handle to the datanode DataNode[] dn = cluster.ListDataNodes(); NUnit.Framework.Assert.IsTrue("There should be only one datanode but found " + dn .Length, dn.Length == 1); LocatedBlocks locations = client.GetNamenode().GetBlockLocations(file1.ToString() , 0, long.MaxValue); IList <LocatedBlock> blocks = locations.GetLocatedBlocks(); // // Create hard links for a few of the blocks // for (int i = 0; i < blocks.Count; i = i + 2) { ExtendedBlock b = blocks[i].GetBlock(); FilePath f = DataNodeTestUtils.GetFile(dn[0], b.GetBlockPoolId(), b.GetLocalBlock ().GetBlockId()); FilePath link = new FilePath(f.ToString() + ".link"); System.Console.Out.WriteLine("Creating hardlink for File " + f + " to " + link); HardLink.CreateHardLink(f, link); } // // Detach all blocks. This should remove hardlinks (if any) // for (int i_1 = 0; i_1 < blocks.Count; i_1++) { ExtendedBlock b = blocks[i_1].GetBlock(); System.Console.Out.WriteLine("testCopyOnWrite detaching block " + b); NUnit.Framework.Assert.IsTrue("Detaching block " + b + " should have returned true" , DataNodeTestUtils.UnlinkBlock(dn[0], b, 1)); } // Since the blocks were already detached earlier, these calls should // return false // for (int i_2 = 0; i_2 < blocks.Count; i_2++) { ExtendedBlock b = blocks[i_2].GetBlock(); System.Console.Out.WriteLine("testCopyOnWrite detaching block " + b); NUnit.Framework.Assert.IsTrue("Detaching block " + b + " should have returned false" , !DataNodeTestUtils.UnlinkBlock(dn[0], b, 1)); } } finally { client.Close(); fs.Close(); cluster.Shutdown(); } }
public virtual void TestMigrateOpenFileToArchival() { Log.Info("testMigrateOpenFileToArchival"); Path fooDir = new Path("/foo"); IDictionary <Path, BlockStoragePolicy> policyMap = Maps.NewHashMap(); policyMap[fooDir] = Cold; TestStorageMover.NamespaceScheme nsScheme = new TestStorageMover.NamespaceScheme( Arrays.AsList(fooDir), null, BlockSize, null, policyMap); TestStorageMover.ClusterScheme clusterScheme = new TestStorageMover.ClusterScheme (DefaultConf, NumDatanodes, Repl, GenStorageTypes(NumDatanodes), null); TestStorageMover.MigrationTest test = new TestStorageMover.MigrationTest(this, clusterScheme , nsScheme); test.SetupCluster(); // create an open file Banner("writing to file /foo/bar"); Path barFile = new Path(fooDir, "bar"); DFSTestUtil.CreateFile(test.dfs, barFile, BlockSize, (short)1, 0L); FSDataOutputStream @out = test.dfs.Append(barFile); @out.WriteBytes("hello, "); ((DFSOutputStream)@out.GetWrappedStream()).Hsync(); try { Banner("start data migration"); test.SetStoragePolicy(); // set /foo to COLD test.Migrate(); // make sure the under construction block has not been migrated LocatedBlocks lbs = test.dfs.GetClient().GetLocatedBlocks(barFile.ToString(), BlockSize ); Log.Info("Locations: " + lbs); IList <LocatedBlock> blks = lbs.GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(1, blks.Count); NUnit.Framework.Assert.AreEqual(1, blks[0].GetLocations().Length); Banner("finish the migration, continue writing"); // make sure the writing can continue @out.WriteBytes("world!"); ((DFSOutputStream)@out.GetWrappedStream()).Hsync(); IOUtils.Cleanup(Log, @out); lbs = test.dfs.GetClient().GetLocatedBlocks(barFile.ToString(), BlockSize); Log.Info("Locations: " + lbs); blks = lbs.GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(1, blks.Count); NUnit.Framework.Assert.AreEqual(1, blks[0].GetLocations().Length); Banner("finish writing, starting reading"); // check the content of /foo/bar FSDataInputStream @in = test.dfs.Open(barFile); byte[] buf = new byte[13]; // read from offset 1024 @in.ReadFully(BlockSize, buf, 0, buf.Length); IOUtils.Cleanup(Log, @in); NUnit.Framework.Assert.AreEqual("hello, world!", Sharpen.Runtime.GetStringForBytes (buf)); } finally { test.ShutdownCluster(); } }
public virtual void TestRestartDfsWithAbandonedBlock() { Configuration conf = new HdfsConfiguration(); // Turn off persistent IPC, so that the DFSClient can survive NN restart conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0); MiniDFSCluster cluster = null; long len = 0; FSDataOutputStream stream; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); FileSystem fs = cluster.GetFileSystem(); // Creating a file with 4096 blockSize to write multiple blocks stream = fs.Create(FilePath, true, BlockSize, (short)1, BlockSize); stream.Write(DataBeforeRestart); stream.Hflush(); // Wait for all of the blocks to get through while (len < BlockSize * (NumBlocks - 1)) { FileStatus status = fs.GetFileStatus(FilePath); len = status.GetLen(); Sharpen.Thread.Sleep(100); } // Abandon the last block DFSClient dfsclient = DFSClientAdapter.GetDFSClient((DistributedFileSystem)fs); HdfsFileStatus fileStatus = dfsclient.GetNamenode().GetFileInfo(FileName); LocatedBlocks blocks = dfsclient.GetNamenode().GetBlockLocations(FileName, 0, BlockSize * NumBlocks); NUnit.Framework.Assert.AreEqual(NumBlocks, blocks.GetLocatedBlocks().Count); LocatedBlock b = blocks.GetLastLocatedBlock(); dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileStatus.GetFileId(), FileName , dfsclient.clientName); // explicitly do NOT close the file. cluster.RestartNameNode(); // Check that the file has no less bytes than before the restart // This would mean that blocks were successfully persisted to the log FileStatus status_1 = fs.GetFileStatus(FilePath); NUnit.Framework.Assert.IsTrue("Length incorrect: " + status_1.GetLen(), status_1. GetLen() == len - BlockSize); // Verify the data showed up from before restart, sans abandoned block. FSDataInputStream readStream = fs.Open(FilePath); try { byte[] verifyBuf = new byte[DataBeforeRestart.Length - BlockSize]; IOUtils.ReadFully(readStream, verifyBuf, 0, verifyBuf.Length); byte[] expectedBuf = new byte[DataBeforeRestart.Length - BlockSize]; System.Array.Copy(DataBeforeRestart, 0, expectedBuf, 0, expectedBuf.Length); Assert.AssertArrayEquals(expectedBuf, verifyBuf); } finally { IOUtils.CloseStream(readStream); } } finally { if (cluster != null) { cluster.Shutdown(); } } }
/* check if there are at least two nodes are on the same rack */ /// <exception cref="System.IO.IOException"/> private void CheckFile(FileSystem fileSys, Path name, int repl) { Configuration conf = fileSys.GetConf(); ClientProtocol namenode = NameNodeProxies.CreateProxy <ClientProtocol>(conf, fileSys .GetUri()).GetProxy(); WaitForBlockReplication(name.ToString(), namenode, Math.Min(numDatanodes, repl), -1); LocatedBlocks locations = namenode.GetBlockLocations(name.ToString(), 0, long.MaxValue ); FileStatus stat = fileSys.GetFileStatus(name); BlockLocation[] blockLocations = fileSys.GetFileBlockLocations(stat, 0L, long.MaxValue ); // verify that rack locations match NUnit.Framework.Assert.IsTrue(blockLocations.Length == locations.LocatedBlockCount ()); for (int i = 0; i < blockLocations.Length; i++) { LocatedBlock blk = locations.Get(i); DatanodeInfo[] datanodes = blk.GetLocations(); string[] topologyPaths = blockLocations[i].GetTopologyPaths(); NUnit.Framework.Assert.IsTrue(topologyPaths.Length == datanodes.Length); for (int j = 0; j < topologyPaths.Length; j++) { bool found = false; for (int k = 0; k < racks.Length; k++) { if (topologyPaths[j].StartsWith(racks[k])) { found = true; break; } } NUnit.Framework.Assert.IsTrue(found); } } bool isOnSameRack = true; bool isNotOnSameRack = true; foreach (LocatedBlock blk_1 in locations.GetLocatedBlocks()) { DatanodeInfo[] datanodes = blk_1.GetLocations(); if (datanodes.Length <= 1) { break; } if (datanodes.Length == 2) { isNotOnSameRack = !(datanodes[0].GetNetworkLocation().Equals(datanodes[1].GetNetworkLocation ())); break; } isOnSameRack = false; isNotOnSameRack = false; for (int i_1 = 0; i_1 < datanodes.Length - 1; i_1++) { Log.Info("datanode " + i_1 + ": " + datanodes[i_1]); bool onRack = false; for (int j = i_1 + 1; j < datanodes.Length; j++) { if (datanodes[i_1].GetNetworkLocation().Equals(datanodes[j].GetNetworkLocation())) { onRack = true; } } if (onRack) { isOnSameRack = true; } if (!onRack) { isNotOnSameRack = true; } if (isOnSameRack && isNotOnSameRack) { break; } } if (!isOnSameRack || !isNotOnSameRack) { break; } } NUnit.Framework.Assert.IsTrue(isOnSameRack); NUnit.Framework.Assert.IsTrue(isNotOnSameRack); }
public virtual void TestExcludeDataNodes() { Configuration conf = WebHdfsTestUtil.CreateConf(); string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1, Rack2, Rack2 }; string[] hosts = new string[] { "DataNode1", "DataNode2", "DataNode3", "DataNode4" , "DataNode5", "DataNode6" }; int nDataNodes = hosts.Length; Log.Info("nDataNodes=" + nDataNodes + ", racks=" + Arrays.AsList(racks) + ", hosts=" + Arrays.AsList(hosts)); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Hosts(hosts).NumDataNodes (nDataNodes).Racks(racks).Build(); try { cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); NameNode namenode = cluster.GetNameNode(); DatanodeManager dm = namenode.GetNamesystem().GetBlockManager().GetDatanodeManager (); Log.Info("dm=" + dm); long blocksize = DFSConfigKeys.DfsBlockSizeDefault; string f = "/foo"; //create a file with three replica. Path p = new Path(f); FSDataOutputStream @out = dfs.Create(p, (short)3); @out.Write(1); @out.Close(); //get replica location. LocatedBlocks locatedblocks = NameNodeAdapter.GetBlockLocations(namenode, f, 0, 1 ); IList <LocatedBlock> lb = locatedblocks.GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(1, lb.Count); DatanodeInfo[] locations = lb[0].GetLocations(); NUnit.Framework.Assert.AreEqual(3, locations.Length); //For GETFILECHECKSUM, OPEN and APPEND, //the chosen datanode must be different with exclude nodes. StringBuilder sb = new StringBuilder(); for (int i = 0; i < 2; i++) { sb.Append(locations[i].GetXferAddr()); { // test GETFILECHECKSUM DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP .Getfilechecksum, -1L, blocksize, sb.ToString()); for (int j = 0; j <= i; j++) { Assert.AssertNotEquals(locations[j].GetHostName(), chosen.GetHostName()); } } { // test OPEN DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP .Open, 0, blocksize, sb.ToString()); for (int j = 0; j <= i; j++) { Assert.AssertNotEquals(locations[j].GetHostName(), chosen.GetHostName()); } } { // test APPEND DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, PostOpParam.OP .Append, -1L, blocksize, sb.ToString()); for (int j = 0; j <= i; j++) { Assert.AssertNotEquals(locations[j].GetHostName(), chosen.GetHostName()); } } sb.Append(","); } } finally { cluster.Shutdown(); } }
public virtual void TestDataLocality() { Configuration conf = WebHdfsTestUtil.CreateConf(); string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1, Rack2, Rack2 }; int nDataNodes = racks.Length; Log.Info("nDataNodes=" + nDataNodes + ", racks=" + Arrays.AsList(racks)); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(nDataNodes ).Racks(racks).Build(); try { cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); NameNode namenode = cluster.GetNameNode(); DatanodeManager dm = namenode.GetNamesystem().GetBlockManager().GetDatanodeManager (); Log.Info("dm=" + dm); long blocksize = DFSConfigKeys.DfsBlockSizeDefault; string f = "/foo"; { //test CREATE for (int i = 0; i < nDataNodes; i++) { //set client address to a particular datanode DataNode dn = cluster.GetDataNodes()[i]; string ipAddr = dm.GetDatanode(dn.GetDatanodeId()).GetIpAddr(); //The chosen datanode must be the same as the client address DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, PutOpParam.OP .Create, -1L, blocksize, null); NUnit.Framework.Assert.AreEqual(ipAddr, chosen.GetIpAddr()); } } //create a file with one replica. Path p = new Path(f); FSDataOutputStream @out = dfs.Create(p, (short)1); @out.Write(1); @out.Close(); //get replica location. LocatedBlocks locatedblocks = NameNodeAdapter.GetBlockLocations(namenode, f, 0, 1 ); IList <LocatedBlock> lb = locatedblocks.GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(1, lb.Count); DatanodeInfo[] locations = lb[0].GetLocations(); NUnit.Framework.Assert.AreEqual(1, locations.Length); DatanodeInfo expected = locations[0]; { //For GETFILECHECKSUM, OPEN and APPEND, //the chosen datanode must be the same as the replica location. //test GETFILECHECKSUM DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP .Getfilechecksum, -1L, blocksize, null); NUnit.Framework.Assert.AreEqual(expected, chosen); } { //test OPEN DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP .Open, 0, blocksize, null); NUnit.Framework.Assert.AreEqual(expected, chosen); } { //test APPEND DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, PostOpParam.OP .Append, -1L, blocksize, null); NUnit.Framework.Assert.AreEqual(expected, chosen); } } finally { cluster.Shutdown(); } }