public static void SetupCluster() { if (DomainSocket.GetLoadingFailureReason() != null) { return; } sockDir = new TemporarySocketDirectory(); HdfsConfiguration conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), "TestParallelShortCircuitReadUnCached._PORT.sock" ).GetAbsolutePath()); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true); // Enabling data transfer encryption should have no effect when using // short-circuit local reads. This is a regression test for HDFS-5353. conf.SetBoolean(DFSConfigKeys.DfsEncryptDataTransferKey, true); conf.SetBoolean(DFSConfigKeys.DfsBlockAccessTokenEnableKey, true); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false); conf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, true); // We want to test reading from stale sockets. conf.SetInt(DFSConfigKeys.DfsDatanodeSocketReuseKeepaliveKey, 1); conf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, 5 * 60 * 1000); conf.SetInt(DFSConfigKeys.DfsClientSocketCacheCapacityKey, 32); // Avoid using the FileInputStreamCache. conf.SetInt(DFSConfigKeys.DfsClientReadShortcircuitStreamsCacheSizeKey, 0); DomainSocket.DisableBindPathValidation(); DFSInputStream.tcpReadsDisabledForTesting = true; SetupCluster(1, conf); }
public static void SetupCluster() { // This is a test of the normal (TCP) read path. For this reason, we turn // off both short-circuit local reads and UNIX domain socket data traffic. HdfsConfiguration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, false); conf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, false); // dfs.domain.socket.path should be ignored because the previous two keys // were set to false. This is a regression test for HDFS-4473. conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "/will/not/be/created"); SetupCluster(DefaultReplicationFactor, conf); }
public static void SetupCluster() { DFSInputStream.tcpReadsDisabledForTesting = true; HdfsConfiguration conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, string.Empty); conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true); conf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, false); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false); conf.Set(DFSConfigKeys.DfsBlockLocalPathAccessUserKey, UserGroupInformation.GetCurrentUser ().GetShortUserName()); DomainSocket.DisableBindPathValidation(); SetupCluster(1, conf); }
public static void SetupCluster() { if (DomainSocket.GetLoadingFailureReason() != null) { return; } DFSInputStream.tcpReadsDisabledForTesting = true; sockDir = new TemporarySocketDirectory(); HdfsConfiguration conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), "TestParallelLocalRead.%d.sock" ).GetAbsolutePath()); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false); DomainSocket.DisableBindPathValidation(); SetupCluster(1, conf); }
/// <exception cref="System.Exception"/> public virtual void TestBlockReaderLocalLegacyWithAppend() { short ReplFactor = 1; HdfsConfiguration conf = GetConfiguration(null); conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); Path path = new Path("/testBlockReaderLocalLegacy"); DFSTestUtil.CreateFile(dfs, path, 10, ReplFactor, 0); DFSTestUtil.WaitReplication(dfs, path, ReplFactor); ClientDatanodeProtocol proxy; Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token; ExtendedBlock originalBlock; long originalGS; { LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString (), 0, 1).Get(0); proxy = DFSUtil.CreateClientDatanodeProtocolProxy(lb.GetLocations()[0], conf, 60000 , false); token = lb.GetBlockToken(); // get block and generation stamp ExtendedBlock blk = new ExtendedBlock(lb.GetBlock()); originalBlock = new ExtendedBlock(blk); originalGS = originalBlock.GetGenerationStamp(); // test getBlockLocalPathInfo BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(blk, token); NUnit.Framework.Assert.AreEqual(originalGS, info.GetBlock().GetGenerationStamp()); } { // append one byte FSDataOutputStream @out = dfs.Append(path); @out.Write(1); @out.Close(); } { // get new generation stamp LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString (), 0, 1).Get(0); long newGS = lb.GetBlock().GetGenerationStamp(); NUnit.Framework.Assert.IsTrue(newGS > originalGS); // getBlockLocalPathInfo using the original block. NUnit.Framework.Assert.AreEqual(originalGS, originalBlock.GetGenerationStamp()); BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(originalBlock, token); NUnit.Framework.Assert.AreEqual(newGS, info.GetBlock().GetGenerationStamp()); } cluster.Shutdown(); }
/// <exception cref="System.IO.IOException"/> private static HdfsConfiguration GetConfiguration(TemporarySocketDirectory socketDir ) { HdfsConfiguration conf = new HdfsConfiguration(); if (socketDir == null) { conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, string.Empty); } else { conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(socketDir.GetDir(), "TestBlockReaderLocalLegacy.%d.sock" ).GetAbsolutePath()); } conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true); conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false); conf.Set(DFSConfigKeys.DfsBlockLocalPathAccessUserKey, UserGroupInformation.GetCurrentUser ().GetShortUserName()); conf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, false); // Set short retry timeouts so this test runs faster conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10); return(conf); }
/// <summary>Tests replication in DFS.</summary> /// <exception cref="System.IO.IOException"/> public virtual void RunReplication(bool simulated) { Configuration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsNamenodeReplicationConsiderloadKey, false); if (simulated) { SimulatedFSDataset.SetFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes ).Racks(racks).Build(); cluster.WaitActive(); IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort()); DFSClient client = new DFSClient(addr, conf); DatanodeInfo[] info = client.DatanodeReport(HdfsConstants.DatanodeReportType.Live ); NUnit.Framework.Assert.AreEqual("Number of Datanodes ", numDatanodes, info.Length ); FileSystem fileSys = cluster.GetFileSystem(); try { Path file1 = new Path("/smallblocktest.dat"); WriteFile(fileSys, file1, 3); CheckFile(fileSys, file1, 3); CleanupFile(fileSys, file1); WriteFile(fileSys, file1, 10); CheckFile(fileSys, file1, 10); CleanupFile(fileSys, file1); WriteFile(fileSys, file1, 4); CheckFile(fileSys, file1, 4); CleanupFile(fileSys, file1); WriteFile(fileSys, file1, 1); CheckFile(fileSys, file1, 1); CleanupFile(fileSys, file1); WriteFile(fileSys, file1, 2); CheckFile(fileSys, file1, 2); CleanupFile(fileSys, file1); } finally { fileSys.Close(); cluster.Shutdown(); } }
public virtual void TestAppendAfterSoftLimit() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1); conf.SetBoolean(DFSConfigKeys.DfsSupportAppendKey, true); //Set small soft-limit for lease long softLimit = 1L; long hardLimit = 9999999L; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.SetLeasePeriod(softLimit, hardLimit); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); FileSystem fs2 = new DistributedFileSystem(); fs2.Initialize(fs.GetUri(), conf); Path testPath = new Path("/testAppendAfterSoftLimit"); byte[] fileContents = AppendTestUtil.InitBuffer(32); // create a new file without closing FSDataOutputStream @out = fs.Create(testPath); @out.Write(fileContents); //Wait for > soft-limit Sharpen.Thread.Sleep(250); try { FSDataOutputStream appendStream2 = fs2.Append(testPath); appendStream2.Write(fileContents); appendStream2.Close(); NUnit.Framework.Assert.AreEqual(fileContents.Length, fs.GetFileStatus(testPath).GetLen ()); } finally { fs.Close(); fs2.Close(); cluster.Shutdown(); } }
/// <exception cref="System.Exception"/> private void TestStatistics(bool isShortCircuit) { Assume.AssumeTrue(DomainSocket.GetLoadingFailureReason() == null); HdfsConfiguration conf = new HdfsConfiguration(); TemporarySocketDirectory sockDir = null; if (isShortCircuit) { DFSInputStream.tcpReadsDisabledForTesting = true; sockDir = new TemporarySocketDirectory(); conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), "TestStatisticsForLocalRead.%d.sock" ).GetAbsolutePath()); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true); DomainSocket.DisableBindPathValidation(); } else { conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, false); } MiniDFSCluster cluster = null; Path TestPath = new Path("/a"); long RandomSeed = 4567L; FSDataInputStream fsIn = null; byte[] original = new byte[TestBlockReaderLocal.BlockReaderLocalTest.TestLength]; FileSystem fs = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestBlockReaderLocal.BlockReaderLocalTest.TestLength , (short)1, RandomSeed); try { DFSTestUtil.WaitReplication(fs, TestPath, (short)1); } catch (Exception e) { NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: " + e); } catch (TimeoutException e) { NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: " + e); } fsIn = fs.Open(TestPath); IOUtils.ReadFully(fsIn, original, 0, TestBlockReaderLocal.BlockReaderLocalTest.TestLength ); HdfsDataInputStream dfsIn = (HdfsDataInputStream)fsIn; NUnit.Framework.Assert.AreEqual(TestBlockReaderLocal.BlockReaderLocalTest.TestLength , dfsIn.GetReadStatistics().GetTotalBytesRead()); NUnit.Framework.Assert.AreEqual(TestBlockReaderLocal.BlockReaderLocalTest.TestLength , dfsIn.GetReadStatistics().GetTotalLocalBytesRead()); if (isShortCircuit) { NUnit.Framework.Assert.AreEqual(TestBlockReaderLocal.BlockReaderLocalTest.TestLength , dfsIn.GetReadStatistics().GetTotalShortCircuitBytesRead()); } else { NUnit.Framework.Assert.AreEqual(0, dfsIn.GetReadStatistics().GetTotalShortCircuitBytesRead ()); } fsIn.Close(); fsIn = null; } finally { DFSInputStream.tcpReadsDisabledForTesting = false; if (fsIn != null) { fsIn.Close(); } if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } if (sockDir != null) { sockDir.Close(); } } }
/// <exception cref="System.IO.IOException"/> public virtual void RunBlockReaderLocalTest(TestBlockReaderLocal.BlockReaderLocalTest test, bool checksum, long readahead) { Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null )); MiniDFSCluster cluster = null; HdfsConfiguration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, !checksum ); conf.SetLong(DFSConfigKeys.DfsBytesPerChecksumKey, TestBlockReaderLocal.BlockReaderLocalTest .BytesPerChecksum); conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C"); conf.SetLong(DFSConfigKeys.DfsClientCacheReadahead, readahead); test.SetConfiguration(conf); FileInputStream dataIn = null; FileInputStream metaIn = null; Path TestPath = new Path("/a"); long RandomSeed = 4567L; BlockReaderLocal blockReaderLocal = null; FSDataInputStream fsIn = null; byte[] original = new byte[TestBlockReaderLocal.BlockReaderLocalTest.TestLength]; FileSystem fs = null; ShortCircuitShm shm = null; RandomAccessFile raf = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestBlockReaderLocal.BlockReaderLocalTest.TestLength , (short)1, RandomSeed); try { DFSTestUtil.WaitReplication(fs, TestPath, (short)1); } catch (Exception e) { NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: " + e); } catch (TimeoutException e) { NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: " + e); } fsIn = fs.Open(TestPath); IOUtils.ReadFully(fsIn, original, 0, TestBlockReaderLocal.BlockReaderLocalTest.TestLength ); fsIn.Close(); fsIn = null; ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestPath); FilePath dataFile = cluster.GetBlockFile(0, block); FilePath metaFile = cluster.GetBlockMetadataFile(0, block); ShortCircuitCache shortCircuitCache = ClientContext.GetFromConf(conf).GetShortCircuitCache (); cluster.Shutdown(); cluster = null; test.Setup(dataFile, checksum); FileInputStream[] streams = new FileInputStream[] { new FileInputStream(dataFile) , new FileInputStream(metaFile) }; dataIn = streams[0]; metaIn = streams[1]; ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId ()); raf = new RandomAccessFile(new FilePath(sockDir.GetDir().GetAbsolutePath(), UUID. RandomUUID().ToString()), "rw"); raf.SetLength(8192); FileInputStream shmStream = new FileInputStream(raf.GetFD()); shm = new ShortCircuitShm(ShortCircuitShm.ShmId.CreateRandom(), shmStream); ShortCircuitReplica replica = new ShortCircuitReplica(key, dataIn, metaIn, shortCircuitCache , Time.Now(), shm.AllocAndRegisterSlot(ExtendedBlockId.FromExtendedBlock(block)) ); blockReaderLocal = new BlockReaderLocal.Builder(new DFSClient.Conf(conf)).SetFilename (TestPath.GetName()).SetBlock(block).SetShortCircuitReplica(replica).SetCachingStrategy (new CachingStrategy(false, readahead)).SetVerifyChecksum(checksum).Build(); dataIn = null; metaIn = null; test.DoTest(blockReaderLocal, original); // BlockReaderLocal should not alter the file position. NUnit.Framework.Assert.AreEqual(0, streams[0].GetChannel().Position()); NUnit.Framework.Assert.AreEqual(0, streams[1].GetChannel().Position()); } finally { if (fsIn != null) { fsIn.Close(); } if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } if (dataIn != null) { dataIn.Close(); } if (metaIn != null) { metaIn.Close(); } if (blockReaderLocal != null) { blockReaderLocal.Close(); } if (shm != null) { shm.Free(); } if (raf != null) { raf.Close(); } } }
public virtual void TestMultipleFilesSmallerThanOneBlock() { Configuration conf = new HdfsConfiguration(); int BlockSize = 6 * 1024; conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize); conf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true); // Make it relinquish locks. When run serially, the result should // be identical. conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); DFSAdmin admin = new DFSAdmin(conf); string nnAddr = conf.Get(DFSConfigKeys.DfsNamenodeHttpAddressKey); string webhdfsuri = WebHdfsFileSystem.Scheme + "://" + nnAddr; System.Console.Out.WriteLine("webhdfsuri=" + webhdfsuri); FileSystem webhdfs = new Path(webhdfsuri).GetFileSystem(conf); try { Path dir = new Path("/test"); bool exceededQuota = false; ContentSummary c; // 1kb file // 6kb block // 192kb quota int FileSize = 1024; int QuotaSize = 32 * (int)fs.GetDefaultBlockSize(dir); NUnit.Framework.Assert.AreEqual(6 * 1024, fs.GetDefaultBlockSize(dir)); NUnit.Framework.Assert.AreEqual(192 * 1024, QuotaSize); // Create the dir and set the quota. We need to enable the quota before // writing the files as setting the quota afterwards will over-write // the cached disk space used for quota verification with the actual // amount used as calculated by INode#spaceConsumedInTree. NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir)); RunCommand(admin, false, "-setSpaceQuota", Sharpen.Extensions.ToString(QuotaSize) , dir.ToString()); // We can create at most 59 files because block allocation is // conservative and initially assumes a full block is used, so we // need to leave at least 3 * BLOCK_SIZE free space when allocating // the last block: (58 * 3 * 1024) (3 * 6 * 1024) = 192kb for (int i = 0; i < 59; i++) { Path file = new Path("/test/test" + i); DFSTestUtil.CreateFile(fs, file, FileSize, (short)3, 1L); DFSTestUtil.WaitReplication(fs, file, (short)3); } // Should account for all 59 files (almost QUOTA_SIZE) c = fs.GetContentSummary(dir); CheckContentSummary(c, webhdfs.GetContentSummary(dir)); NUnit.Framework.Assert.AreEqual("Invalid space consumed", 59 * FileSize * 3, c.GetSpaceConsumed ()); NUnit.Framework.Assert.AreEqual("Invalid space consumed", QuotaSize - (59 * FileSize * 3), 3 * (fs.GetDefaultBlockSize(dir) - FileSize)); // Now check that trying to create another file violates the quota try { Path file = new Path("/test/test59"); DFSTestUtil.CreateFile(fs, file, FileSize, (short)3, 1L); DFSTestUtil.WaitReplication(fs, file, (short)3); } catch (QuotaExceededException) { exceededQuota = true; } NUnit.Framework.Assert.IsTrue("Quota not exceeded", exceededQuota); NUnit.Framework.Assert.AreEqual(2, cluster.GetNamesystem().GetFSDirectory().GetYieldCount ()); } finally { cluster.Shutdown(); } }
public virtual void TestBlockAllocationAdjustsUsageConservatively() { Configuration conf = new HdfsConfiguration(); int BlockSize = 6 * 1024; conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize); conf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); DFSAdmin admin = new DFSAdmin(conf); string nnAddr = conf.Get(DFSConfigKeys.DfsNamenodeHttpAddressKey); string webhdfsuri = WebHdfsFileSystem.Scheme + "://" + nnAddr; System.Console.Out.WriteLine("webhdfsuri=" + webhdfsuri); FileSystem webhdfs = new Path(webhdfsuri).GetFileSystem(conf); try { Path dir = new Path("/test"); Path file1 = new Path("/test/test1"); Path file2 = new Path("/test/test2"); bool exceededQuota = false; int QuotaSize = 3 * BlockSize; // total space usage including // repl. int FileSize = BlockSize / 2; ContentSummary c; // Create the directory and set the quota NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir)); RunCommand(admin, false, "-setSpaceQuota", Sharpen.Extensions.ToString(QuotaSize) , dir.ToString()); // Creating a file should use half the quota DFSTestUtil.CreateFile(fs, file1, FileSize, (short)3, 1L); DFSTestUtil.WaitReplication(fs, file1, (short)3); c = fs.GetContentSummary(dir); CheckContentSummary(c, webhdfs.GetContentSummary(dir)); NUnit.Framework.Assert.AreEqual("Quota is half consumed", QuotaSize / 2, c.GetSpaceConsumed ()); // We can not create the 2nd file because even though the total spaced // used by two files (2 * 3 * 512/2) would fit within the quota (3 * 512) // when a block for a file is created the space used is adjusted // conservatively (3 * block size, ie assumes a full block is written) // which will violate the quota (3 * block size) since we've already // used half the quota for the first file. try { DFSTestUtil.CreateFile(fs, file2, FileSize, (short)3, 1L); } catch (QuotaExceededException) { exceededQuota = true; } NUnit.Framework.Assert.IsTrue("Quota not exceeded", exceededQuota); } finally { cluster.Shutdown(); } }
public virtual void TestReadSelectNonStaleDatanode() { HdfsConfiguration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForReadKey, true); long staleInterval = 30 * 1000 * 60; conf.SetLong(DFSConfigKeys.DfsNamenodeStaleDatanodeIntervalKey, staleInterval); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes ).Racks(racks).Build(); cluster.WaitActive(); IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort()); DFSClient client = new DFSClient(addr, conf); IList <DatanodeDescriptor> nodeInfoList = cluster.GetNameNode().GetNamesystem().GetBlockManager ().GetDatanodeManager().GetDatanodeListForReport(HdfsConstants.DatanodeReportType .Live); NUnit.Framework.Assert.AreEqual("Unexpected number of datanodes", numDatanodes, nodeInfoList .Count); FileSystem fileSys = cluster.GetFileSystem(); FSDataOutputStream stm = null; try { // do the writing but do not close the FSDataOutputStream // in order to mimic the ongoing writing Path fileName = new Path("/file1"); stm = fileSys.Create(fileName, true, fileSys.GetConf().GetInt(CommonConfigurationKeys .IoFileBufferSizeKey, 4096), (short)3, blockSize); stm.Write(new byte[(blockSize * 3) / 2]); // We do not close the stream so that // the writing seems to be still ongoing stm.Hflush(); LocatedBlocks blocks = client.GetNamenode().GetBlockLocations(fileName.ToString() , 0, blockSize); DatanodeInfo[] nodes = blocks.Get(0).GetLocations(); NUnit.Framework.Assert.AreEqual(nodes.Length, 3); DataNode staleNode = null; DatanodeDescriptor staleNodeInfo = null; // stop the heartbeat of the first node staleNode = this.StopDataNodeHeartbeat(cluster, nodes[0].GetHostName()); NUnit.Framework.Assert.IsNotNull(staleNode); // set the first node as stale staleNodeInfo = cluster.GetNameNode().GetNamesystem().GetBlockManager().GetDatanodeManager ().GetDatanode(staleNode.GetDatanodeId()); DFSTestUtil.ResetLastUpdatesWithOffset(staleNodeInfo, -(staleInterval + 1)); LocatedBlocks blocksAfterStale = client.GetNamenode().GetBlockLocations(fileName. ToString(), 0, blockSize); DatanodeInfo[] nodesAfterStale = blocksAfterStale.Get(0).GetLocations(); NUnit.Framework.Assert.AreEqual(nodesAfterStale.Length, 3); NUnit.Framework.Assert.AreEqual(nodesAfterStale[2].GetHostName(), nodes[0].GetHostName ()); // restart the staleNode's heartbeat DataNodeTestUtils.SetHeartbeatsDisabledForTests(staleNode, false); // reset the first node as non-stale, so as to avoid two stale nodes DFSTestUtil.ResetLastUpdatesWithOffset(staleNodeInfo, 0); LocatedBlock lastBlock = client.GetLocatedBlocks(fileName.ToString(), 0, long.MaxValue ).GetLastLocatedBlock(); nodes = lastBlock.GetLocations(); NUnit.Framework.Assert.AreEqual(nodes.Length, 3); // stop the heartbeat of the first node for the last block staleNode = this.StopDataNodeHeartbeat(cluster, nodes[0].GetHostName()); NUnit.Framework.Assert.IsNotNull(staleNode); // set the node as stale DatanodeDescriptor dnDesc = cluster.GetNameNode().GetNamesystem().GetBlockManager ().GetDatanodeManager().GetDatanode(staleNode.GetDatanodeId()); DFSTestUtil.ResetLastUpdatesWithOffset(dnDesc, -(staleInterval + 1)); LocatedBlock lastBlockAfterStale = client.GetLocatedBlocks(fileName.ToString(), 0 , long.MaxValue).GetLastLocatedBlock(); nodesAfterStale = lastBlockAfterStale.GetLocations(); NUnit.Framework.Assert.AreEqual(nodesAfterStale.Length, 3); NUnit.Framework.Assert.AreEqual(nodesAfterStale[2].GetHostName(), nodes[0].GetHostName ()); } finally { if (stm != null) { stm.Close(); } client.Close(); cluster.Shutdown(); } }
/// <summary>Tests all FsEditLogOps that are converted to inotify events.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Hdfs.Inotify.MissingEventsException"/> public virtual void TestBasic() { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize); conf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true); // so that we can get an atime change conf.SetLong(DFSConfigKeys.DfsNamenodeAccesstimePrecisionKey, 1); MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf); builder.GetDfsBuilder().NumDataNodes(2); MiniQJMHACluster cluster = builder.Build(); try { cluster.GetDfsCluster().WaitActive(); cluster.GetDfsCluster().TransitionToActive(0); DFSClient client = new DFSClient(cluster.GetDfsCluster().GetNameNode(0).GetNameNodeAddress (), conf); FileSystem fs = cluster.GetDfsCluster().GetFileSystem(0); DFSTestUtil.CreateFile(fs, new Path("/file"), BlockSize, (short)1, 0L); DFSTestUtil.CreateFile(fs, new Path("/file3"), BlockSize, (short)1, 0L); DFSTestUtil.CreateFile(fs, new Path("/file5"), BlockSize, (short)1, 0L); DFSInotifyEventInputStream eis = client.GetInotifyEventStream(); client.Rename("/file", "/file4", null); // RenameOp -> RenameEvent client.Rename("/file4", "/file2"); // RenameOldOp -> RenameEvent // DeleteOp, AddOp -> UnlinkEvent, CreateEvent OutputStream os = client.Create("/file2", true, (short)2, BlockSize); os.Write(new byte[BlockSize]); os.Close(); // CloseOp -> CloseEvent // AddOp -> AppendEvent os = client.Append("/file2", BlockSize, EnumSet.Of(CreateFlag.Append), null, null ); os.Write(new byte[BlockSize]); os.Close(); // CloseOp -> CloseEvent Sharpen.Thread.Sleep(10); // so that the atime will get updated on the next line client.Open("/file2").Read(new byte[1]); // TimesOp -> MetadataUpdateEvent // SetReplicationOp -> MetadataUpdateEvent client.SetReplication("/file2", (short)1); // ConcatDeleteOp -> AppendEvent, UnlinkEvent, CloseEvent client.Concat("/file2", new string[] { "/file3" }); client.Delete("/file2", false); // DeleteOp -> UnlinkEvent client.Mkdirs("/dir", null, false); // MkdirOp -> CreateEvent // SetPermissionsOp -> MetadataUpdateEvent client.SetPermission("/dir", FsPermission.ValueOf("-rw-rw-rw-")); // SetOwnerOp -> MetadataUpdateEvent client.SetOwner("/dir", "username", "groupname"); client.CreateSymlink("/dir", "/dir2", false); // SymlinkOp -> CreateEvent client.SetXAttr("/file5", "user.field", Sharpen.Runtime.GetBytesForString("value" ), EnumSet.Of(XAttrSetFlag.Create)); // SetXAttrOp -> MetadataUpdateEvent // RemoveXAttrOp -> MetadataUpdateEvent client.RemoveXAttr("/file5", "user.field"); // SetAclOp -> MetadataUpdateEvent client.SetAcl("/file5", AclEntry.ParseAclSpec("user::rwx,user:foo:rw-,group::r--,other::---" , true)); client.RemoveAcl("/file5"); // SetAclOp -> MetadataUpdateEvent client.Rename("/file5", "/dir"); // RenameOldOp -> RenameEvent EventBatch batch = null; // RenameOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); long txid = batch.GetTxid(); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Rename); Event.RenameEvent re = (Event.RenameEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.AreEqual("/file4", re.GetDstPath()); NUnit.Framework.Assert.AreEqual("/file", re.GetSrcPath()); NUnit.Framework.Assert.IsTrue(re.GetTimestamp() > 0); long eventsBehind = eis.GetTxidsBehindEstimate(); // RenameOldOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Rename); Event.RenameEvent re2 = (Event.RenameEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(re2.GetDstPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(re2.GetSrcPath().Equals("/file4")); NUnit.Framework.Assert.IsTrue(re.GetTimestamp() > 0); // AddOp with overwrite batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Create); Event.CreateEvent ce = (Event.CreateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(ce.GetiNodeType() == Event.CreateEvent.INodeType.File ); NUnit.Framework.Assert.IsTrue(ce.GetPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(ce.GetCtime() > 0); NUnit.Framework.Assert.IsTrue(ce.GetReplication() > 0); NUnit.Framework.Assert.IsTrue(ce.GetSymlinkTarget() == null); NUnit.Framework.Assert.IsTrue(ce.GetOverwrite()); NUnit.Framework.Assert.AreEqual(BlockSize, ce.GetDefaultBlockSize()); // CloseOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Close); Event.CloseEvent ce2 = (Event.CloseEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(ce2.GetPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(ce2.GetFileSize() > 0); NUnit.Framework.Assert.IsTrue(ce2.GetTimestamp() > 0); // AppendOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Append); Event.AppendEvent append2 = (Event.AppendEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.AreEqual("/file2", append2.GetPath()); NUnit.Framework.Assert.IsFalse(append2.ToNewBlock()); // CloseOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Close); NUnit.Framework.Assert.IsTrue(((Event.CloseEvent)batch.GetEvents()[0]).GetPath(). Equals("/file2")); // TimesOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue.GetPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(mue.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Times); // SetReplicationOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue2 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue2.GetPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(mue2.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Replication); NUnit.Framework.Assert.IsTrue(mue2.GetReplication() == 1); // ConcatDeleteOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(3, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Append); NUnit.Framework.Assert.IsTrue(((Event.AppendEvent)batch.GetEvents()[0]).GetPath() .Equals("/file2")); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[1].GetEventType() == Event.EventType .Unlink); Event.UnlinkEvent ue2 = (Event.UnlinkEvent)batch.GetEvents()[1]; NUnit.Framework.Assert.IsTrue(ue2.GetPath().Equals("/file3")); NUnit.Framework.Assert.IsTrue(ue2.GetTimestamp() > 0); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[2].GetEventType() == Event.EventType .Close); Event.CloseEvent ce3 = (Event.CloseEvent)batch.GetEvents()[2]; NUnit.Framework.Assert.IsTrue(ce3.GetPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(ce3.GetTimestamp() > 0); // DeleteOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Unlink); Event.UnlinkEvent ue = (Event.UnlinkEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(ue.GetPath().Equals("/file2")); NUnit.Framework.Assert.IsTrue(ue.GetTimestamp() > 0); // MkdirOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Create); Event.CreateEvent ce4 = (Event.CreateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(ce4.GetiNodeType() == Event.CreateEvent.INodeType.Directory ); NUnit.Framework.Assert.IsTrue(ce4.GetPath().Equals("/dir")); NUnit.Framework.Assert.IsTrue(ce4.GetCtime() > 0); NUnit.Framework.Assert.IsTrue(ce4.GetReplication() == 0); NUnit.Framework.Assert.IsTrue(ce4.GetSymlinkTarget() == null); // SetPermissionsOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue3 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue3.GetPath().Equals("/dir")); NUnit.Framework.Assert.IsTrue(mue3.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Perms); NUnit.Framework.Assert.IsTrue(mue3.GetPerms().ToString().Contains("rw-rw-rw-")); // SetOwnerOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue4 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue4.GetPath().Equals("/dir")); NUnit.Framework.Assert.IsTrue(mue4.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Owner); NUnit.Framework.Assert.IsTrue(mue4.GetOwnerName().Equals("username")); NUnit.Framework.Assert.IsTrue(mue4.GetGroupName().Equals("groupname")); // SymlinkOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Create); Event.CreateEvent ce5 = (Event.CreateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(ce5.GetiNodeType() == Event.CreateEvent.INodeType.Symlink ); NUnit.Framework.Assert.IsTrue(ce5.GetPath().Equals("/dir2")); NUnit.Framework.Assert.IsTrue(ce5.GetCtime() > 0); NUnit.Framework.Assert.IsTrue(ce5.GetReplication() == 0); NUnit.Framework.Assert.IsTrue(ce5.GetSymlinkTarget().Equals("/dir")); // SetXAttrOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue5 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue5.GetPath().Equals("/file5")); NUnit.Framework.Assert.IsTrue(mue5.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Xattrs); NUnit.Framework.Assert.IsTrue(mue5.GetxAttrs().Count == 1); NUnit.Framework.Assert.IsTrue(mue5.GetxAttrs()[0].GetName().Contains("field")); NUnit.Framework.Assert.IsTrue(!mue5.IsxAttrsRemoved()); // RemoveXAttrOp batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue6 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue6.GetPath().Equals("/file5")); NUnit.Framework.Assert.IsTrue(mue6.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Xattrs); NUnit.Framework.Assert.IsTrue(mue6.GetxAttrs().Count == 1); NUnit.Framework.Assert.IsTrue(mue6.GetxAttrs()[0].GetName().Contains("field")); NUnit.Framework.Assert.IsTrue(mue6.IsxAttrsRemoved()); // SetAclOp (1) batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue7 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue7.GetPath().Equals("/file5")); NUnit.Framework.Assert.IsTrue(mue7.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Acls); NUnit.Framework.Assert.IsTrue(mue7.GetAcls().Contains(AclEntry.ParseAclEntry("user::rwx" , true))); // SetAclOp (2) batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Metadata); Event.MetadataUpdateEvent mue8 = (Event.MetadataUpdateEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(mue8.GetPath().Equals("/file5")); NUnit.Framework.Assert.IsTrue(mue8.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType .Acls); NUnit.Framework.Assert.IsTrue(mue8.GetAcls() == null); // RenameOp (2) batch = WaitForNextEvents(eis); NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length); txid = CheckTxid(batch, txid); NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType .Rename); Event.RenameEvent re3 = (Event.RenameEvent)batch.GetEvents()[0]; NUnit.Framework.Assert.IsTrue(re3.GetDstPath().Equals("/dir/file5")); NUnit.Framework.Assert.IsTrue(re3.GetSrcPath().Equals("/file5")); NUnit.Framework.Assert.IsTrue(re.GetTimestamp() > 0); // Returns null when there are no further events NUnit.Framework.Assert.IsTrue(eis.Poll() == null); // make sure the estimate hasn't changed since the above assertion // tells us that we are fully caught up to the current namesystem state // and we should not have been behind at all when eventsBehind was set // either, since there were few enough events that they should have all // been read to the client during the first poll() call NUnit.Framework.Assert.IsTrue(eis.GetTxidsBehindEstimate() == eventsBehind); } finally { cluster.Shutdown(); } }