/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestQueryAfterRestart() { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); // start rolling upgrade dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare); QueryForPreparation(dfs); dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); dfs.SaveNamespace(); dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); cluster.RestartNameNodes(); dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Query); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> public virtual void TestLogAndRestart() { conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, MiniDFSCluster.GetBaseDirectory() + "/TestNNWithQJM/image"); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, mjc.GetQuorumJournalURI("myjournal" ).ToString()); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs (false).Build(); try { cluster.GetFileSystem().Mkdirs(TestPath); // Restart the NN and make sure the edit was persisted // and loaded again cluster.RestartNameNode(); NUnit.Framework.Assert.IsTrue(cluster.GetFileSystem().Exists(TestPath)); cluster.GetFileSystem().Mkdirs(TestPath2); // Restart the NN again and make sure both edits are persisted. cluster.RestartNameNode(); NUnit.Framework.Assert.IsTrue(cluster.GetFileSystem().Exists(TestPath)); NUnit.Framework.Assert.IsTrue(cluster.GetFileSystem().Exists(TestPath2)); } finally { cluster.Shutdown(); } }
public virtual void TestDoGetShouldCloseTheDFSInputStreamIfResponseGetOutPutStreamThrowsAnyException () { MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(1).Build(); try { Path testFile = CreateFile(); SetUpForDoGetTest(cluster, testFile); Org.Mockito.Mockito.DoThrow(new IOException()).When(mockHttpServletResponse).GetOutputStream (); DFSInputStream fsMock = Org.Mockito.Mockito.Mock <DFSInputStream>(); Org.Mockito.Mockito.DoReturn(fsMock).When(clientMock).Open(testFile.ToString()); Org.Mockito.Mockito.DoReturn(Sharpen.Extensions.ValueOf(4)).When(fsMock).GetFileLength (); try { sfile.DoGet(mockHttpServletRequest, mockHttpServletResponse); NUnit.Framework.Assert.Fail("Not throwing the IOException"); } catch (IOException) { Org.Mockito.Mockito.Verify(clientMock, Org.Mockito.Mockito.AtLeastOnce()).Close(); } } finally { cluster.Shutdown(); } }
public virtual void TestRaceWhileNNStartup() { MiniDFSCluster cluster = null; Configuration conf = WebHdfsTestUtil.CreateConf(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); cluster.WaitActive(); NameNode namenode = cluster.GetNameNode(); NamenodeProtocols rpcServer = namenode.GetRpcServer(); Whitebox.SetInternalState(namenode, "rpcServer", null); Path foo = new Path("/foo"); FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem .Scheme); try { webHdfs.Mkdirs(foo); NUnit.Framework.Assert.Fail("Expected RetriableException"); } catch (RetriableException e) { GenericTestUtils.AssertExceptionContains("Namenode is in startup mode", e); } Whitebox.SetInternalState(namenode, "rpcServer", rpcServer); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestRollbackCommand() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; Path foo = new Path("/foo"); Path bar = new Path("/bar"); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); DFSAdmin dfsadmin = new DFSAdmin(conf); dfs.Mkdirs(foo); // start rolling upgrade dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); NUnit.Framework.Assert.AreEqual(0, dfsadmin.Run(new string[] { "-rollingUpgrade", "prepare" })); dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); // create new directory dfs.Mkdirs(bar); // check NNStorage NNStorage storage = cluster.GetNamesystem().GetFSImage().GetStorage(); CheckNNStorage(storage, 3, -1); } finally { // (startSegment, mkdir, endSegment) if (cluster != null) { cluster.Shutdown(); } } NameNode nn = null; try { nn = NameNode.CreateNameNode(new string[] { "-rollingUpgrade", "rollback" }, conf ); // make sure /foo is still there, but /bar is not INode fooNode = nn.GetNamesystem().GetFSDirectory().GetINode4Write(foo.ToString() ); NUnit.Framework.Assert.IsNotNull(fooNode); INode barNode = nn.GetNamesystem().GetFSDirectory().GetINode4Write(bar.ToString() ); NUnit.Framework.Assert.IsNull(barNode); // check the details of NNStorage NNStorage storage = nn.GetNamesystem().GetFSImage().GetStorage(); // (startSegment, upgrade marker, mkdir, endSegment) CheckNNStorage(storage, 3, 7); } finally { if (nn != null) { nn.Stop(); nn.Join(); } } }
public virtual void TestLease() { MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); try { FileSystem fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir)); Path a = new Path(dir, "a"); Path b = new Path(dir, "b"); DataOutputStream a_out = fs.Create(a); a_out.WriteBytes("something"); NUnit.Framework.Assert.IsTrue(HasLease(cluster, a)); NUnit.Framework.Assert.IsTrue(!HasLease(cluster, b)); DataOutputStream b_out = fs.Create(b); b_out.WriteBytes("something"); NUnit.Framework.Assert.IsTrue(HasLease(cluster, a)); NUnit.Framework.Assert.IsTrue(HasLease(cluster, b)); a_out.Close(); b_out.Close(); NUnit.Framework.Assert.IsTrue(!HasLease(cluster, a)); NUnit.Framework.Assert.IsTrue(!HasLease(cluster, b)); fs.Delete(dir, true); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary> /// Test for catching "no datanode" IOException, when to create a file /// but datanode is not running for some reason. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestCreateWithNoDN() { MiniDFSCluster cluster = null; Configuration conf = WebHdfsTestUtil.CreateConf(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1); cluster.WaitActive(); FileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme ); fs.Create(new Path("/testnodatanode")); NUnit.Framework.Assert.Fail("No exception was thrown"); } catch (IOException ex) { GenericTestUtils.AssertExceptionContains("Failed to find datanode", ex); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestNfsUpgrade() { MiniDFSCluster cluster = null; FileSystem fs = null; try { cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(0).Build(); FilePath sharedDir = new FilePath(cluster.GetSharedEditsDir(0, 1)); // No upgrade is in progress at the moment. CheckClusterPreviousDirExistence(cluster, false); AssertCTimesEqual(cluster); CheckPreviousDirExistence(sharedDir, false); // Transition NN0 to active and do some FS ops. cluster.TransitionToActive(0); fs = HATestUtil.ConfigureFailoverFs(cluster, conf); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo1"))); // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade // flag. cluster.ShutdownNameNode(1); cluster.GetNameNodeInfos()[0].SetStartOpt(HdfsServerConstants.StartupOption.Upgrade ); cluster.RestartNameNode(0, false); CheckNnPreviousDirExistence(cluster, 0, true); CheckNnPreviousDirExistence(cluster, 1, false); CheckPreviousDirExistence(sharedDir, true); // NN0 should come up in the active state when given the -upgrade option, // so no need to transition it to active. NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo2"))); // Restart NN0 without the -upgrade flag, to make sure that works. cluster.GetNameNodeInfos()[0].SetStartOpt(HdfsServerConstants.StartupOption.Regular ); cluster.RestartNameNode(0, false); // Make sure we can still do FS ops after upgrading. cluster.TransitionToActive(0); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo3"))); // Now bootstrap the standby with the upgraded info. int rc = BootstrapStandby.Run(new string[] { "-force" }, cluster.GetConfiguration (1)); NUnit.Framework.Assert.AreEqual(0, rc); // Now restart NN1 and make sure that we can do ops against that as well. cluster.RestartNameNode(1); cluster.TransitionToStandby(0); cluster.TransitionToActive(1); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo4"))); AssertCTimesEqual(cluster); } finally { if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestStartingWithUpgradeInProgressSucceeds() { MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(0).Build(); // Simulate an upgrade having started. for (int i = 0; i < 2; i++) { foreach (URI uri in cluster.GetNameDirs(i)) { FilePath prevTmp = new FilePath(new FilePath(uri), Storage.StorageTmpPrevious); Log.Info("creating previous tmp dir: " + prevTmp); NUnit.Framework.Assert.IsTrue(prevTmp.Mkdirs()); } } cluster.RestartNameNodes(); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>Bring up two clusters and assert that they are in different directories. /// </summary> /// <exception cref="System.Exception">on a failure</exception> public virtual void TestDualClusters() { FilePath testDataCluster2 = new FilePath(testDataPath, Cluster2); FilePath testDataCluster3 = new FilePath(testDataPath, Cluster3); Configuration conf = new HdfsConfiguration(); string c2Path = testDataCluster2.GetAbsolutePath(); conf.Set(MiniDFSCluster.HdfsMinidfsBasedir, c2Path); MiniDFSCluster cluster2 = new MiniDFSCluster.Builder(conf).Build(); MiniDFSCluster cluster3 = null; try { string dataDir2 = cluster2.GetDataDirectory(); NUnit.Framework.Assert.AreEqual(new FilePath(c2Path + "/data"), new FilePath(dataDir2 )); //change the data dir conf.Set(MiniDFSCluster.HdfsMinidfsBasedir, testDataCluster3.GetAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); cluster3 = builder.Build(); string dataDir3 = cluster3.GetDataDirectory(); NUnit.Framework.Assert.IsTrue("Clusters are bound to the same directory: " + dataDir2 , !dataDir2.Equals(dataDir3)); } finally { MiniDFSCluster.ShutdownCluster(cluster3); MiniDFSCluster.ShutdownCluster(cluster2); } }
public virtual void TestMkdirRpcNonCanonicalPath() { MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); try { NamenodeProtocols nnrpc = cluster.GetNameNodeRpc(); foreach (string pathStr in NonCanonicalPaths) { try { nnrpc.Mkdirs(pathStr, new FsPermission((short)0x1ed), true); NUnit.Framework.Assert.Fail("Did not fail when called with a non-canonicalized path: " + pathStr); } catch (InvalidPathException) { } } } finally { // expected cluster.Shutdown(); } }
/// <summary>Test the updation of NeededReplications for the Appended Block</summary> /// <exception cref="System.Exception"/> public virtual void TestUpdateNeededReplicationsForAppendedFile() { Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); DistributedFileSystem fileSystem = null; try { // create a file. fileSystem = cluster.GetFileSystem(); Path f = new Path("/testAppend"); FSDataOutputStream create = fileSystem.Create(f, (short)2); create.Write(Sharpen.Runtime.GetBytesForString("/testAppend")); create.Close(); // Append to the file. FSDataOutputStream append = fileSystem.Append(f); append.Write(Sharpen.Runtime.GetBytesForString("/testAppend")); append.Close(); // Start a new datanode cluster.StartDataNodes(conf, 1, true, null, null); // Check for replications DFSTestUtil.WaitReplication(fileSystem, f, (short)2); } finally { if (null != fileSystem) { fileSystem.Close(); } cluster.Shutdown(); } }
public virtual void Test() { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); FSNamesystem fsn = cluster.GetNameNode().namesystem; MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer(); ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState" ); string snapshotStats = (string)(mbs.GetAttribute(mxbeanName, "SnapshotStats")); IDictionary <string, object> stat = (IDictionary <string, object>)JSON.Parse(snapshotStats ); NUnit.Framework.Assert.IsTrue(stat.Contains("SnapshottableDirectories") && (long) stat["SnapshottableDirectories"] == fsn.GetNumSnapshottableDirs()); NUnit.Framework.Assert.IsTrue(stat.Contains("Snapshots") && (long)stat["Snapshots" ] == fsn.GetNumSnapshots()); object pendingDeletionBlocks = mbs.GetAttribute(mxbeanName, "PendingDeletionBlocks" ); NUnit.Framework.Assert.IsNotNull(pendingDeletionBlocks); NUnit.Framework.Assert.IsTrue(pendingDeletionBlocks is long); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestWithFSNamesystemWriteLock() { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; FSNamesystem fsn = null; try { cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); fsn = cluster.GetNameNode().namesystem; fsn.WriteLock(); TestFSNamesystemMBean.MBeanClient client = new TestFSNamesystemMBean.MBeanClient( ); client.Start(); client.Join(20000); NUnit.Framework.Assert.IsTrue("JMX calls are blocked when FSNamesystem's writerlock" + "is owned by another thread", client.succeeded); client.Interrupt(); } finally { if (fsn != null && fsn.HasWriteLock()) { fsn.WriteUnlock(); } if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestHSyncBlockBoundary() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = cluster.GetFileSystem(); Path p = new Path("/testHSyncBlockBoundary/foo"); int len = 1 << 16; byte[] fileContents = AppendTestUtil.InitBuffer(len); FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag .Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)1, len, null); // fill exactly one block (tests the SYNC_BLOCK case) and flush @out.Write(fileContents, 0, len); @out.Hflush(); // the full block should have caused a sync CheckSyncMetric(cluster, 1); @out.Hsync(); // first on block again CheckSyncMetric(cluster, 1); // write one more byte and sync again @out.Write(1); @out.Hsync(); CheckSyncMetric(cluster, 2); @out.Close(); CheckSyncMetric(cluster, 3); cluster.Shutdown(); }
public virtual void TestMultipleNamespacesConfigured() { Configuration conf = DFSTestUtil.NewHAConfiguration(LogicalName); MiniDFSCluster cluster = null; WebHdfsFileSystem fs = null; try { cluster = new MiniDFSCluster.Builder(conf).NnTopology(topo).NumDataNodes(1).Build (); HATestUtil.SetFailoverConfigurations(cluster, conf, LogicalName); cluster.WaitActive(); DFSTestUtil.AddHAConfiguration(conf, LogicalName + "remote"); DFSTestUtil.SetFakeHttpAddresses(conf, LogicalName + "remote"); fs = (WebHdfsFileSystem)FileSystem.Get(WebhdfsUri, conf); NUnit.Framework.Assert.AreEqual(2, fs.GetResolvedNNAddr().Length); } finally { IOUtils.Cleanup(null, fs); if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestFinalizedReplicas() { // bring up a cluster of 3 Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024L); conf.SetInt(DFSConfigKeys.DfsClientWritePacketSizeKey, 512); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); try { // test finalized replicas string TopDir = "/test"; DFSTestUtil util = new DFSTestUtil.Builder().SetName("TestDatanodeRestart").SetNumFiles (2).Build(); util.CreateFiles(fs, TopDir, (short)3); util.WaitReplication(fs, TopDir, (short)3); util.CheckFiles(fs, TopDir); cluster.RestartDataNodes(); cluster.WaitActive(); util.CheckFiles(fs, TopDir); } finally { cluster.Shutdown(); } }
public virtual void TestHA() { Configuration conf = DFSTestUtil.NewHAConfiguration(LogicalName); MiniDFSCluster cluster = null; FileSystem fs = null; try { cluster = new MiniDFSCluster.Builder(conf).NnTopology(topo).NumDataNodes(0).Build (); HATestUtil.SetFailoverConfigurations(cluster, conf, LogicalName); cluster.WaitActive(); fs = FileSystem.Get(WebhdfsUri, conf); cluster.TransitionToActive(0); Path dir = new Path("/test"); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir)); cluster.ShutdownNameNode(0); cluster.TransitionToActive(1); Path dir2 = new Path("/test2"); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir2)); } finally { IOUtils.Cleanup(null, fs); if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.Exception"/> public virtual void TestLargeDirectory() { Configuration conf = WebHdfsTestUtil.CreateConf(); int listLimit = 2; // force small chunking of directory listing conf.SetInt(DFSConfigKeys.DfsListLimit, listLimit); // force paths to be only owner-accessible to ensure ugi isn't changing // during listStatus FsPermission.SetUMask(conf, new FsPermission((short)0x3f)); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); try { cluster.WaitActive(); WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme).SetPermission (new Path("/"), new FsPermission(FsAction.All, FsAction.All, FsAction.All)); // trick the NN into not believing it's not the superuser so we can // tell if the correct user is used by listStatus UserGroupInformation.SetLoginUser(UserGroupInformation.CreateUserForTesting("not-superuser" , new string[] { "not-supergroup" })); UserGroupInformation.CreateUserForTesting("me", new string[] { "my-group" }).DoAs (new _PrivilegedExceptionAction_263(conf, listLimit)); } finally { cluster.Shutdown(); } }
public virtual void TestSecureHAToken() { Configuration conf = DFSTestUtil.NewHAConfiguration(LogicalName); conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); MiniDFSCluster cluster = null; WebHdfsFileSystem fs = null; try { cluster = new MiniDFSCluster.Builder(conf).NnTopology(topo).NumDataNodes(0).Build (); HATestUtil.SetFailoverConfigurations(cluster, conf, LogicalName); cluster.WaitActive(); fs = Org.Mockito.Mockito.Spy((WebHdfsFileSystem)FileSystem.Get(WebhdfsUri, conf)); FileSystemTestHelper.AddFileSystemForTesting(WebhdfsUri, conf, fs); cluster.TransitionToActive(0); Org.Apache.Hadoop.Security.Token.Token <object> token = ((Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>)fs.GetDelegationToken(null)); cluster.ShutdownNameNode(0); cluster.TransitionToActive(1); token.Renew(conf); token.Cancel(conf); Org.Mockito.Mockito.Verify(fs).RenewDelegationToken(token); Org.Mockito.Mockito.Verify(fs).CancelDelegationToken(token); } finally { IOUtils.Cleanup(null, fs); if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestWebHdfsRenameSnapshot() { MiniDFSCluster cluster = null; Configuration conf = WebHdfsTestUtil.CreateConf(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem .Scheme); Path foo = new Path("/foo"); dfs.Mkdirs(foo); dfs.AllowSnapshot(foo); webHdfs.CreateSnapshot(foo, "s1"); Path s1path = SnapshotTestHelper.GetSnapshotRoot(foo, "s1"); NUnit.Framework.Assert.IsTrue(webHdfs.Exists(s1path)); // rename s1 to s2 webHdfs.RenameSnapshot(foo, "s1", "s2"); NUnit.Framework.Assert.IsFalse(webHdfs.Exists(s1path)); Path s2path = SnapshotTestHelper.GetSnapshotRoot(foo, "s2"); NUnit.Framework.Assert.IsTrue(webHdfs.Exists(s2path)); webHdfs.DeleteSnapshot(foo, "s2"); NUnit.Framework.Assert.IsFalse(webHdfs.Exists(s2path)); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestBlockIdGeneration() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); try { cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); // Create a file that is 10 blocks long. Path path = new Path("testBlockIdGeneration.dat"); DFSTestUtil.CreateFile(fs, path, IoSize, BlockSize * 10, BlockSize, Replication, Seed); IList <LocatedBlock> blocks = DFSTestUtil.GetAllBlocks(fs, path); Log.Info("Block0 id is " + blocks[0].GetBlock().GetBlockId()); long nextBlockExpectedId = blocks[0].GetBlock().GetBlockId() + 1; // Ensure that the block IDs are sequentially increasing. for (int i = 1; i < blocks.Count; ++i) { long nextBlockId = blocks[i].GetBlock().GetBlockId(); Log.Info("Block" + i + " id is " + nextBlockId); Assert.AssertThat(nextBlockId, CoreMatchers.Is(nextBlockExpectedId)); ++nextBlockExpectedId; } } finally { cluster.Shutdown(); } }
public virtual void TestDTInInsecureCluster() { MiniDFSCluster cluster = null; Configuration conf = WebHdfsTestUtil.CreateConf(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem .Scheme); webHdfs.GetDelegationToken(null); NUnit.Framework.Assert.Fail("No exception is thrown."); } catch (AccessControlException ace) { NUnit.Framework.Assert.IsTrue(ace.Message.StartsWith(WebHdfsFileSystem.CantFallbackToInsecureMsg )); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> internal static void Setrep(int fromREP, int toREP, bool simulatedStorage) { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { SimulatedFSDataset.SetFactory(conf); } conf.Set(DFSConfigKeys.DfsReplicationKey, string.Empty + fromREP); conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L); conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString (2)); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(10).Build( ); FileSystem fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem ); try { Path root = TestDFSShell.Mkdir(fs, new Path("/test/setrep" + fromREP + "-" + toREP )); Path f = TestDFSShell.WriteFile(fs, new Path(root, "foo")); { // Verify setrep for changing replication string[] args = new string[] { "-setrep", "-w", string.Empty + toREP, string.Empty + f }; FsShell shell = new FsShell(); shell.SetConf(conf); try { NUnit.Framework.Assert.AreEqual(0, shell.Run(args)); } catch (Exception e) { NUnit.Framework.Assert.IsTrue("-setrep " + e, false); } } //get fs again since the old one may be closed fs = cluster.GetFileSystem(); FileStatus file = fs.GetFileStatus(f); long len = file.GetLen(); foreach (BlockLocation locations in fs.GetFileBlockLocations(file, 0, len)) { NUnit.Framework.Assert.IsTrue(locations.GetHosts().Length == toREP); } TestDFSShell.Show("done setrep waiting: " + root); } finally { try { fs.Close(); } catch (Exception) { } cluster.Shutdown(); } }
//cluster.shutdown(); /// <exception cref="System.Exception"/> public virtual void TestMismatchedNNIsRejected() { conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, MiniDFSCluster.GetBaseDirectory() + "/TestNNWithQJM/image"); string defaultEditsDir = conf.Get(DFSConfigKeys.DfsNamenodeEditsDirKey); conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, mjc.GetQuorumJournalURI("myjournal" ).ToString()); // Start a NN, so the storage is formatted -- both on-disk // and QJM. MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs (false).Build(); cluster.Shutdown(); // Reformat just the on-disk portion Configuration onDiskOnly = new Configuration(conf); onDiskOnly.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, defaultEditsDir); NameNode.Format(onDiskOnly); // Start the NN - should fail because the JNs are still formatted // with the old namespace ID. try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs(false ).Format(false).Build(); NUnit.Framework.Assert.Fail("New NN with different namespace should have been rejected" ); } catch (IOException ioe) { GenericTestUtils.AssertExceptionContains("Unable to start log segment 1: too few journals" , ioe); } }
public virtual void TestSequenceFileSync() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = cluster.GetFileSystem(); Path p = new Path("/testSequenceFileSync/foo"); int len = 1 << 16; FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag .Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)1, len, null); SequenceFile.Writer w = SequenceFile.CreateWriter(new Configuration(), SequenceFile.Writer .Stream(@out), SequenceFile.Writer.KeyClass(typeof(RandomDatum)), SequenceFile.Writer .ValueClass(typeof(RandomDatum)), SequenceFile.Writer.Compression(SequenceFile.CompressionType .None, new DefaultCodec())); w.Hflush(); CheckSyncMetric(cluster, 0); w.Hsync(); CheckSyncMetric(cluster, 1); int seed = new Random().Next(); RandomDatum.Generator generator = new RandomDatum.Generator(seed); generator.Next(); w.Append(generator.GetKey(), generator.GetValue()); w.Hsync(); CheckSyncMetric(cluster, 2); w.Close(); CheckSyncMetric(cluster, 2); @out.Close(); CheckSyncMetric(cluster, 3); cluster.Shutdown(); }
/// <exception cref="System.Exception"/> public virtual void DoEncryptionTest(int numMappers, int numReducers, int numNodes , int numLines, bool isUber) { MiniDFSCluster dfsCluster = null; MiniMRClientCluster mrCluster = null; FileSystem fileSystem = null; try { Configuration conf = new Configuration(); // Start the mini-MR and mini-DFS clusters dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numNodes).Build(); fileSystem = dfsCluster.GetFileSystem(); mrCluster = MiniMRClientClusterFactory.Create(this.GetType(), numNodes, conf); // Generate input. CreateInput(fileSystem, numMappers, numLines); // Run the test. RunMergeTest(new JobConf(mrCluster.GetConfig()), fileSystem, numMappers, numReducers , numLines, isUber); } finally { if (dfsCluster != null) { dfsCluster.Shutdown(); } if (mrCluster != null) { mrCluster.Stop(); } } }
public virtual void TestHSyncWithReplication() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); FileSystem fs = cluster.GetFileSystem(); Path p = new Path("/testHSyncWithReplication/foo"); int len = 1 << 16; FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag .Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)3, len, null); @out.Write(1); @out.Hflush(); CheckSyncMetric(cluster, 0, 0); CheckSyncMetric(cluster, 1, 0); CheckSyncMetric(cluster, 2, 0); @out.Hsync(); CheckSyncMetric(cluster, 0, 1); CheckSyncMetric(cluster, 1, 1); CheckSyncMetric(cluster, 2, 1); @out.Hsync(); CheckSyncMetric(cluster, 0, 2); CheckSyncMetric(cluster, 1, 2); CheckSyncMetric(cluster, 2, 2); cluster.Shutdown(); }
public virtual void TestFailoverWithBK() { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, BKJMUtil.CreateJournalURI("/hotfailover" ).ToString()); BKJMUtil.AddJournalManagerDefinition(conf); cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(0).ManageNameDfsSharedDirs(false).Build(); NameNode nn1 = cluster.GetNameNode(0); NameNode nn2 = cluster.GetNameNode(1); cluster.WaitActive(); cluster.TransitionToActive(0); Path p = new Path("/testBKJMfailover"); FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf); fs.Mkdirs(p); cluster.ShutdownNameNode(0); cluster.TransitionToActive(1); NUnit.Framework.Assert.IsTrue(fs.Exists(p)); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestDFSAdminDatanodeUpgradeControlCommands() { // start a cluster Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); DFSAdmin dfsadmin = new DFSAdmin(conf); DataNode dn = cluster.GetDataNodes()[0]; // check the datanode string dnAddr = dn.GetDatanodeId().GetIpcAddr(false); string[] args1 = new string[] { "-getDatanodeInfo", dnAddr }; RunCmd(dfsadmin, true, args1); // issue shutdown to the datanode. string[] args2 = new string[] { "-shutdownDatanode", dnAddr, "upgrade" }; RunCmd(dfsadmin, true, args2); // the datanode should be down. Sharpen.Thread.Sleep(2000); NUnit.Framework.Assert.IsFalse("DataNode should exit", dn.IsDatanodeUp()); // ping should fail. NUnit.Framework.Assert.AreEqual(-1, dfsadmin.Run(args1)); } finally { if (cluster != null) { cluster.Shutdown(); } } }