public virtual void SetupCluster() { conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); HAUtil.SetAllowStandbyReads(conf, true); fsHelper = new FileSystemTestHelper(); string testRoot = fsHelper.GetTestRootDir(); testRootDir = new FilePath(testRoot).GetAbsoluteFile(); conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName + "://file" + new Path(testRootDir.ToString(), "test.jks").ToUri()); cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(1).Build(); cluster.WaitActive(); cluster.TransitionToActive(0); fs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs(cluster, conf); DFSTestUtil.CreateKey(TestKey, cluster, 0, conf); DFSTestUtil.CreateKey(TestKey, cluster, 1, conf); nn0 = cluster.GetNameNode(0); nn1 = cluster.GetNameNode(1); dfsAdmin0 = new HdfsAdmin(cluster.GetURI(0), conf); dfsAdmin1 = new HdfsAdmin(cluster.GetURI(1), conf); KeyProviderCryptoExtension nn0Provider = cluster.GetNameNode(0).GetNamesystem().GetProvider (); fs.GetClient().SetKeyProvider(nn0Provider); }
public static void ClusterSetupAtBeginning() { cluster = new MiniDFSCluster.Builder(clusterConf).NnTopology(MiniDFSNNTopology.SimpleFederatedTopology (2)).NumDataNodes(2).Build(); cluster.WaitClusterUp(); fc = FileContext.GetFileContext(cluster.GetURI(0), clusterConf); fc2 = FileContext.GetFileContext(cluster.GetURI(1), clusterConf); }
public static void ClusterSetupAtBeginning() { clusterConf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true); cluster = new MiniDFSCluster.Builder(clusterConf).NnTopology(MiniDFSNNTopology.SimpleFederatedTopology (2)).NumDataNodes(2).Build(); cluster.WaitClusterUp(); fc = FileContext.GetFileContext(cluster.GetURI(0), clusterConf); fc2 = FileContext.GetFileContext(cluster.GetURI(1), clusterConf); }
public static void ClusterSetupAtBegining() { SupportsBlocks = true; Conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(2).Build(); cluster.WaitClusterUp(); fc = FileContext.GetFileContext(cluster.GetURI(0), Conf); }
public static void ClusterSetupAtBegining() { Configuration conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); fc = FileContext.GetFileContext(cluster.GetURI(0), conf); defaultWorkingDirectory = fc.MakeQualified(new Path("/user/" + UserGroupInformation .GetCurrentUser().GetShortUserName())); fc.Mkdir(defaultWorkingDirectory, FileContext.DefaultPerm, true); }
/// <exception cref="System.IO.IOException"/> private void RestartCluster(bool format) { if (cluster != null) { cluster.Shutdown(); cluster = null; } cluster = new MiniDFSCluster.Builder(Conf).Format(format).Build(); cluster.WaitClusterUp(); fc = FileContext.GetFileContext(cluster.GetURI(0), Conf); }
public virtual void TestIPFailoverProxyProviderLogicalUri() { // setup the config with the IP failover proxy provider class Configuration config = new HdfsConfiguration(conf); URI nnUri = cluster.GetURI(0); config.Set(DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." + nnUri.GetHost (), typeof(IPFailoverProxyProvider).FullName); NUnit.Framework.Assert.IsFalse("IPFailoverProxyProvider should not use logical URI." , HAUtil.UseLogicalUri(config, nnUri)); }
public static void TestSetUp() { conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsListLimit, 2); cluster = new MiniDFSCluster.Builder(conf).Build(); fs = cluster.GetFileSystem(); fc = FileContext.GetFileContext(cluster.GetURI(0), conf); hftpfs = cluster.GetHftpFileSystem(0); dfsClient = new DFSClient(NameNode.GetAddress(conf), conf); file1 = new Path("filestatus.dat"); WriteFile(fs, file1, 1, fileSize, blockSize); }
public static void ClusterSetupAtBegining() { SupportsBlocks = true; Conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(2).Build(); cluster.WaitClusterUp(); fc = FileContext.GetFileContext(cluster.GetURI(0), Conf); Path defaultWorkingDirectory = fc.MakeQualified(new Path("/user/" + UserGroupInformation .GetCurrentUser().GetShortUserName())); fc.Mkdir(defaultWorkingDirectory, FileContext.DefaultPerm, true); }
/// <summary>Regression test for HDFS-3357.</summary> /// <remarks> /// Regression test for HDFS-3357. Check that the datanode is respecting /// its configured keepalive timeout. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestDatanodeRespectsKeepAliveTimeout() { Configuration clientConf = new Configuration(conf); // Set a client socket cache expiry time much longer than // the datanode-side expiration time. long ClientExpiryMs = 60000L; clientConf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, ClientExpiryMs ); clientConf.Set(DFSConfigKeys.DfsClientContext, "testDatanodeRespectsKeepAliveTimeout" ); DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI() , clientConf); PeerCache peerCache = ClientContext.GetFromConf(clientConf).GetPeerCache(); DFSTestUtil.CreateFile(fs, TestFile, 1L, (short)1, 0L); // Clients that write aren't currently re-used. NUnit.Framework.Assert.AreEqual(0, peerCache.Size()); AssertXceiverCount(0); // Reads the file, so we should get a // cached socket, and should have an xceiver on the other side. DFSTestUtil.ReadFile(fs, TestFile); NUnit.Framework.Assert.AreEqual(1, peerCache.Size()); AssertXceiverCount(1); // Sleep for a bit longer than the keepalive timeout // and make sure the xceiver died. Sharpen.Thread.Sleep(DFSConfigKeys.DfsDatanodeSocketReuseKeepaliveDefault + 50); AssertXceiverCount(0); // The socket is still in the cache, because we don't // notice that it's closed until we try to read // from it again. NUnit.Framework.Assert.AreEqual(1, peerCache.Size()); // Take it out of the cache - reading should // give an EOF. Peer peer = peerCache.Get(dn.GetDatanodeId(), false); NUnit.Framework.Assert.IsNotNull(peer); NUnit.Framework.Assert.AreEqual(-1, peer.GetInputStream().Read()); }
public virtual void TestBinaryTokenFile() { Configuration conf = mrCluster.GetConfig(); // provide namenodes names for the job to get the delegation tokens for string nnUri = dfsCluster.GetURI(0).ToString(); conf.Set(MRJobConfig.JobNamenodes, nnUri + "," + nnUri); // using argument to pass the file name string[] args = new string[] { "-m", "1", "-r", "1", "-mt", "1", "-rt", "1" }; int res = -1; try { res = ToolRunner.Run(conf, new TestBinaryTokenFile.MySleepJob(this), args); } catch (Exception e) { System.Console.Out.WriteLine("Job failed with " + e.GetLocalizedMessage()); Sharpen.Runtime.PrintStackTrace(e, System.Console.Out); NUnit.Framework.Assert.Fail("Job failed"); } NUnit.Framework.Assert.AreEqual("dist job res is not 0:", 0, res); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Javax.Security.Auth.Login.LoginException"/> private static void RestartCluster() { if (cluster != null) { cluster.Shutdown(); cluster = null; } cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(1).Format(false).Build(); cluster.WaitClusterUp(); fc = FileContext.GetFileContext(cluster.GetURI(0), Conf); defaultWorkingDirectory = fc.MakeQualified(new Path("/user/" + UserGroupInformation .GetCurrentUser().GetShortUserName())); fc.Mkdir(defaultWorkingDirectory, FileContext.DefaultPerm, true); }
public virtual void Setup() { conf = new HdfsConfiguration(); fsHelper = new FileSystemTestHelper(); // Set up java key store string testRoot = fsHelper.GetTestRootDir(); FilePath testRootDir = new FilePath(testRoot).GetAbsoluteFile(); Path jksPath = new Path(testRootDir.ToString(), "test.jks"); conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName + "://file" + jksPath.ToUri()); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); Logger.GetLogger(typeof(EncryptionZoneManager)).SetLevel(Level.Trace); fs = cluster.GetFileSystem(); fsWrapper = new FileSystemTestWrapper(cluster.GetFileSystem()); fcWrapper = new FileContextTestWrapper(FileContext.GetFileContext(cluster.GetURI( ), conf)); dfsAdmin = new HdfsAdmin(cluster.GetURI(), conf); // Need to set the client's KeyProvider to the NN's for JKS, // else the updates do not get flushed properly fs.GetClient().SetKeyProvider(cluster.GetNameNode().GetNamesystem().GetProvider() ); DFSTestUtil.CreateKey(TestKey, cluster, conf); }
/// <summary>Tests DataTransferProtocol with the given client configuration.</summary> /// <param name="conf">client configuration</param> /// <exception cref="System.IO.IOException">if there is an I/O error</exception> private void DoTest(HdfsConfiguration conf) { fs = FileSystem.Get(cluster.GetURI(), conf); FileSystemTestHelper.CreateFile(fs, Path, NumBlocks, BlockSize); Assert.AssertArrayEquals(FileSystemTestHelper.GetFileData(NumBlocks, BlockSize), Sharpen.Runtime.GetBytesForString(DFSTestUtil.ReadFile(fs, Path), "UTF-8")); BlockLocation[] blockLocations = fs.GetFileBlockLocations(Path, 0, long.MaxValue); NUnit.Framework.Assert.IsNotNull(blockLocations); NUnit.Framework.Assert.AreEqual(NumBlocks, blockLocations.Length); foreach (BlockLocation blockLocation in blockLocations) { NUnit.Framework.Assert.IsNotNull(blockLocation.GetHosts()); NUnit.Framework.Assert.AreEqual(3, blockLocation.GetHosts().Length); } }
/// <exception cref="System.Exception"/> public TestFSRMStateStoreTester(TestFSRMStateStore _enclosing, MiniDFSCluster cluster , bool adminCheckEnable) { this._enclosing = _enclosing; Path workingDirPath = new Path("/yarn/Test"); this.adminCheckEnable = adminCheckEnable; this.cluster = cluster; FileSystem fs = cluster.GetFileSystem(); fs.Mkdirs(workingDirPath); Path clusterURI = new Path(cluster.GetURI()); this.workingDirPathURI = new Path(clusterURI, workingDirPath); fs.Close(); }
public static void Setup() { string currentUser = Runtime.GetProperty("user.name"); config.Set("fs.permissions.umask-mode", "u=rwx,g=,o="); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey (currentUser), "*"); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey (currentUser), "*"); fsHelper = new FileSystemTestHelper(); // Set up java key store string testRoot = fsHelper.GetTestRootDir(); testRootDir = new FilePath(testRoot).GetAbsoluteFile(); Path jksPath = new Path(testRootDir.ToString(), "test.jks"); config.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName + "://file" + jksPath.ToUri()); ProxyUsers.RefreshSuperUserGroupsConfiguration(config); cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build(); cluster.WaitActive(); hdfs = cluster.GetFileSystem(); nn = cluster.GetNameNode(); dfsAdmin = new HdfsAdmin(cluster.GetURI(), config); // Use ephemeral ports in case tests are running in parallel config.SetInt("nfs3.mountd.port", 0); config.SetInt("nfs3.server.port", 0); // Start NFS with allowed.hosts set to "* rw" config.Set("dfs.nfs.exports.allowed.hosts", "* rw"); nfs = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3(config); nfs.StartServiceInternal(false); nfsd = (RpcProgramNfs3)nfs.GetRpcProgram(); hdfs.GetClient().SetKeyProvider(nn.GetNamesystem().GetProvider()); DFSTestUtil.CreateKey(TestKey, cluster, config); // Mock SecurityHandler which returns system user.name securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>(); Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(currentUser); // Mock SecurityHandler which returns a dummy username "harry" securityHandlerUnpriviledged = Org.Mockito.Mockito.Mock <SecurityHandler>(); Org.Mockito.Mockito.When(securityHandlerUnpriviledged.GetUser()).ThenReturn("harry" ); }
protected override URI TestURI() { return(cluster.GetURI(0)); }