public virtual void TestCanReadData() { Path file1 = new Path("/fileToRead.dat"); Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; FileSystem fileSys = null; BackupNode backup = null; try { // Start NameNode and BackupNode cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(true).Build(); fileSys = cluster.GetFileSystem(); long txid = cluster.GetNameNodeRpc().GetTransactionID(); backup = StartBackupNode(conf, HdfsServerConstants.StartupOption.Backup, 1); WaitCheckpointDone(cluster, txid); // Setup dual NameNode configuration for DataNodes string rpcAddrKeyPreffix = DFSConfigKeys.DfsNamenodeRpcAddressKey + ".bnCluster"; string nnAddr = cluster.GetNameNode().GetNameNodeAddressHostPortString(); conf.Get(DFSConfigKeys.DfsNamenodeRpcAddressKey); string bnAddr = backup.GetNameNodeAddressHostPortString(); conf.Set(DFSConfigKeys.DfsNameservices, "bnCluster"); conf.Set(DFSConfigKeys.DfsNameserviceId, "bnCluster"); conf.Set(DFSConfigKeys.DfsHaNamenodesKeyPrefix + ".bnCluster", "nnActive, nnBackup" ); conf.Set(rpcAddrKeyPreffix + ".nnActive", nnAddr); conf.Set(rpcAddrKeyPreffix + ".nnBackup", bnAddr); cluster.StartDataNodes(conf, 3, true, HdfsServerConstants.StartupOption.Regular, null); DFSTestUtil.CreateFile(fileSys, file1, 8192, (short)3, 0); // Read the same file from file systems pointing to NN and BN FileSystem bnFS = FileSystem.Get(new Path("hdfs://" + bnAddr).ToUri(), conf); string nnData = DFSTestUtil.ReadFile(fileSys, file1); string bnData = DFSTestUtil.ReadFile(bnFS, file1); NUnit.Framework.Assert.AreEqual("Data read from BackupNode and NameNode is not the same." , nnData, bnData); } catch (IOException e) { Log.Error("Error in TestBackupNode: ", e); NUnit.Framework.Assert.IsTrue(e.GetLocalizedMessage(), false); } finally { if (fileSys != null) { fileSys.Close(); } if (backup != null) { backup.Stop(); } if (cluster != null) { cluster.Shutdown(); } } }
public static void SetUpClass() { coreSitePath = "." + FilePath.separator + "target" + FilePath.separator + "test-classes" + FilePath.separator + "core-site.xml"; Configuration conf = new HdfsConfiguration(); Configuration conf2 = new HdfsConfiguration(); dfsCluster = new MiniDFSCluster.Builder(conf).Build(); conf2.Set(MiniDFSCluster.HdfsMinidfsBasedir, conf.Get(MiniDFSCluster.HdfsMinidfsBasedir ) + "_2"); dfsCluster2 = new MiniDFSCluster.Builder(conf2).Build(); }
static PipelineAck() { OobTimeout = new long[NumOobTypes]; HdfsConfiguration conf = new HdfsConfiguration(); string[] ele = conf.Get(DFSConfigKeys.DfsDatanodeOobTimeoutKey, DFSConfigKeys.DfsDatanodeOobTimeoutDefault ).Split(","); for (int i = 0; i < NumOobTypes; i++) { OobTimeout[i] = (i < ele.Length) ? long.Parse(ele[i]) : 0; } }
public override void SetConf(Configuration conf) { conf = new HdfsConfiguration(conf); string nameNodePrincipal = conf.Get(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey , string.Empty); if (Log.IsDebugEnabled()) { Log.Debug("Using NN principal: " + nameNodePrincipal); } conf.Set(CommonConfigurationKeys.HadoopSecurityServiceUserNameKey, nameNodePrincipal ); base.SetConf(conf); }
/// <summary> /// Add the requisite security principal settings to the given Configuration, /// returning a copy. /// </summary> /// <param name="conf">the original config</param> /// <returns>a copy with the security settings added</returns> public static Configuration AddSecurityConfiguration(Configuration conf) { // Make a copy so we don't mutate it. Also use an HdfsConfiguration to // force loading of hdfs-site.xml. conf = new HdfsConfiguration(conf); string nameNodePrincipal = conf.Get(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey , string.Empty); if (Log.IsDebugEnabled()) { Log.Debug("Using NN principal: " + nameNodePrincipal); } conf.Set(CommonConfigurationKeys.HadoopSecurityServiceUserNameKey, nameNodePrincipal ); return(conf); }