/// <exception cref="System.Exception"/> public virtual void TestSetrepIncWithUnderReplicatedBlocks() { // 1 min timeout Configuration conf = new HdfsConfiguration(); short ReplicationFactor = 2; string FileName = "/testFile"; Path FilePath = new Path(FileName); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(ReplicationFactor + 1).Build(); try { // create a file with one block with a replication factor of 2 FileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, FilePath, 1L, ReplicationFactor, 1L); DFSTestUtil.WaitReplication(fs, FilePath, ReplicationFactor); // remove one replica from the blocksMap so block becomes under-replicated // but the block does not get put into the under-replicated blocks queue BlockManager bm = cluster.GetNamesystem().GetBlockManager(); ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, FilePath); DatanodeDescriptor dn = bm.blocksMap.GetStorages(b.GetLocalBlock()).GetEnumerator ().Next().GetDatanodeDescriptor(); bm.AddToInvalidates(b.GetLocalBlock(), dn); Sharpen.Thread.Sleep(5000); bm.blocksMap.RemoveNode(b.GetLocalBlock(), dn); // increment this file's replication factor FsShell shell = new FsShell(conf); NUnit.Framework.Assert.AreEqual(0, shell.Run(new string[] { "-setrep", "-w", Sharpen.Extensions.ToString (1 + ReplicationFactor), FileName })); } finally { cluster.Shutdown(); } }
private void Execute(string[] args, string namenode) { FsShell shell = new FsShell(); FileSystem fs = null; try { ToolRunner.Run(shell, args); fs = FileSystem.Get(NameNode.GetUri(NameNode.GetAddress(namenode)), shell.GetConf ()); NUnit.Framework.Assert.IsTrue("Directory does not get created", fs.IsDirectory(new Path("/data"))); fs.Delete(new Path("/data"), true); } catch (Exception e) { System.Console.Error.WriteLine(e.Message); Sharpen.Runtime.PrintStackTrace(e); } finally { if (fs != null) { try { fs.Close(); } catch (IOException) { } } } }
public static void Setup() { conf.SetClass("fs.file.impl", typeof(TestFsShellReturnCode.LocalFileSystemExtn), typeof(LocalFileSystem)); fileSys = FileSystem.Get(conf); fsShell = new FsShell(conf); }
/// <exception cref="System.Exception"/> public virtual void TestRmWithNonexistentGlob() { Configuration conf = new Configuration(); FsShell shell = new FsShell(); shell.SetConf(conf); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); TextWriter err = new TextWriter(bytes); TextWriter oldErr = System.Console.Error; Runtime.SetErr(err); string results; try { int exit = shell.Run(new string[] { "-rm", "nomatch*" }); Assert.Equal(1, exit); results = bytes.ToString(); Assert.True(results.Contains("rm: `nomatch*': No such file or directory" )); } finally { IOUtils.CloseStream(err); Runtime.SetErr(oldErr); } }
/// <exception cref="System.IO.IOException"/> internal static void Setrep(int fromREP, int toREP, bool simulatedStorage) { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { SimulatedFSDataset.SetFactory(conf); } conf.Set(DFSConfigKeys.DfsReplicationKey, string.Empty + fromREP); conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L); conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString (2)); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(10).Build( ); FileSystem fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem ); try { Path root = TestDFSShell.Mkdir(fs, new Path("/test/setrep" + fromREP + "-" + toREP )); Path f = TestDFSShell.WriteFile(fs, new Path(root, "foo")); { // Verify setrep for changing replication string[] args = new string[] { "-setrep", "-w", string.Empty + toREP, string.Empty + f }; FsShell shell = new FsShell(); shell.SetConf(conf); try { NUnit.Framework.Assert.AreEqual(0, shell.Run(args)); } catch (Exception e) { NUnit.Framework.Assert.IsTrue("-setrep " + e, false); } } //get fs again since the old one may be closed fs = cluster.GetFileSystem(); FileStatus file = fs.GetFileStatus(f); long len = file.GetLen(); foreach (BlockLocation locations in fs.GetFileBlockLocations(file, 0, len)) { NUnit.Framework.Assert.IsTrue(locations.GetHosts().Length == toREP); } TestDFSShell.Show("done setrep waiting: " + root); } finally { try { fs.Close(); } catch (Exception) { } cluster.Shutdown(); } }
public virtual void TestRenameSnapshotCommandWithIllegalArguments() { ByteArrayOutputStream @out = new ByteArrayOutputStream(); TextWriter psOut = new TextWriter(@out); Runtime.SetOut(psOut); Runtime.SetErr(psOut); FsShell shell = new FsShell(); shell.SetConf(conf); string[] argv1 = new string[] { "-renameSnapshot", "/tmp", "s1" }; int val = shell.Run(argv1); NUnit.Framework.Assert.IsTrue(val == -1); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains(argv1[0] + ": Incorrect number of arguments." )); @out.Reset(); string[] argv2 = new string[] { "-renameSnapshot", "/tmp", "s1", "s2", "s3" }; val = shell.Run(argv2); NUnit.Framework.Assert.IsTrue(val == -1); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains(argv2[0] + ": Incorrect number of arguments." )); psOut.Close(); @out.Close(); }
/// <summary> /// Adding as part of jira HDFS-5343 /// Test for checking the cat command on snapshot path it /// cannot read a file beyond snapshot file length /// </summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotFileLengthWithCatCommand() { FSDataInputStream fis = null; FileStatus fileStatus = null; int bytesRead; byte[] buffer = new byte[Blocksize * 8]; hdfs.Mkdirs(sub); Path file1 = new Path(sub, file1Name); DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, Seed); hdfs.AllowSnapshot(sub); hdfs.CreateSnapshot(sub, snapshot1); DFSTestUtil.AppendFile(hdfs, file1, Blocksize); // Make sure we can read the entire file via its non-snapshot path. fileStatus = hdfs.GetFileStatus(file1); NUnit.Framework.Assert.AreEqual("Unexpected file length", Blocksize * 2, fileStatus .GetLen()); fis = hdfs.Open(file1); bytesRead = fis.Read(buffer, 0, buffer.Length); NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize * 2, bytesRead ); fis.Close(); Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name); fis = hdfs.Open(file1snap1); fileStatus = hdfs.GetFileStatus(file1snap1); NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), Blocksize); // Make sure we can only read up to the snapshot length. bytesRead = fis.Read(buffer, 0, buffer.Length); NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize, bytesRead); fis.Close(); TextWriter outBackup = System.Console.Out; TextWriter errBackup = System.Console.Error; ByteArrayOutputStream bao = new ByteArrayOutputStream(); Runtime.SetOut(new TextWriter(bao)); Runtime.SetErr(new TextWriter(bao)); // Make sure we can cat the file upto to snapshot length FsShell shell = new FsShell(); try { ToolRunner.Run(conf, shell, new string[] { "-cat", "/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1" }); NUnit.Framework.Assert.AreEqual("Unexpected # bytes from -cat", Blocksize, bao.Size ()); } finally { Runtime.SetOut(outBackup); Runtime.SetErr(errBackup); } }
/// <exception cref="System.Exception"/> internal static string ExecCmd(FsShell shell, string[] args) { ByteArrayOutputStream baout = new ByteArrayOutputStream(); TextWriter @out = new TextWriter(baout, true); TextWriter old = System.Console.Out; Runtime.SetOut(@out); int ret = shell.Run(args); @out.Close(); Runtime.SetOut(old); return(ret.ToString()); }
/// <exception cref="System.Exception"/> public virtual void Execute(Configuration conf, FileSystem fs) { fs.Mkdirs(new Path(TestFsShellPermission.TestRoot)); this._enclosing.CreateFiles(fs, TestFsShellPermission.TestRoot, this.fileEntries); FsShell fsShell = new FsShell(conf); string deletePath = TestFsShellPermission.TestRoot + "/" + this.deleteEntry.GetPath (); string[] tmpCmdOpts = StringUtils.Split(this.cmdAndOptions); AList <string> tmpArray = new AList <string>(Arrays.AsList(tmpCmdOpts)); tmpArray.AddItem(deletePath); string[] cmdOpts = Sharpen.Collections.ToArray(tmpArray, new string[tmpArray.Count ]); this.userUgi.DoAs(new _PrivilegedExceptionAction_153(fsShell, cmdOpts)); bool deleted = !fs.Exists(new Path(deletePath)); NUnit.Framework.Assert.AreEqual(this.expectedToDelete, deleted); TestFsShellPermission.Deldir(fs, TestFsShellPermission.TestRoot); }
/// <exception cref="System.Exception"/> public virtual void TestGetWithInvalidSourcePathShouldNotDisplayNullInConsole() { Configuration conf = new Configuration(); FsShell shell = new FsShell(); shell.SetConf(conf); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); TextWriter @out = new TextWriter(bytes); TextWriter oldErr = System.Console.Error; Runtime.SetErr(@out); string results; try { Path tdir = new Path(TestRootDir, "notNullCopy"); fileSys.Delete(tdir, true); fileSys.Mkdirs(tdir); string[] args = new string[3]; args[0] = "-get"; args[1] = new Path(tdir.ToUri().GetPath(), "/invalidSrc").ToString(); args[2] = new Path(tdir.ToUri().GetPath(), "/invalidDst").ToString(); Assert.True("file exists", !fileSys.Exists(new Path(args[1]))); Assert.True("file exists", !fileSys.Exists(new Path(args[2]))); int run = shell.Run(args); results = bytes.ToString(); Assert.Equal("Return code should be 1", 1, run); Assert.True(" Null is coming when source path is invalid. ", !results .Contains("get: null")); Assert.True(" Not displaying the intended message ", results.Contains ("get: `" + args[1] + "': No such file or directory")); } finally { IOUtils.CloseStream(@out); Runtime.SetErr(oldErr); } }
/// <exception cref="System.Exception"/> public virtual void TestRmForceWithNonexistentGlob() { Configuration conf = new Configuration(); FsShell shell = new FsShell(); shell.SetConf(conf); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); TextWriter err = new TextWriter(bytes); TextWriter oldErr = System.Console.Error; Runtime.SetErr(err); try { int exit = shell.Run(new string[] { "-rm", "-f", "nomatch*" }); Assert.Equal(0, exit); Assert.True(bytes.ToString().IsEmpty()); } finally { IOUtils.CloseStream(err); Runtime.SetErr(oldErr); } }
/// <exception cref="System.Exception"/> public virtual void TestInvalidDefaultFS() { // if default fs doesn't exist or is invalid, but the path provided in // arguments is valid - fsshell should work FsShell shell = new FsShell(); Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, "hhhh://doesnotexist/"); shell.SetConf(conf); string[] args = new string[2]; args[0] = "-ls"; args[1] = "file:///"; // this is valid, so command should run int res = shell.Run(args); System.Console.Out.WriteLine("res =" + res); shell.SetConf(conf); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); TextWriter @out = new TextWriter(bytes); TextWriter oldErr = System.Console.Error; Runtime.SetErr(@out); string results; try { int run = shell.Run(args); results = bytes.ToString(); Log.Info("result=" + results); Assert.True("Return code should be 0", run == 0); } finally { IOUtils.CloseStream(@out); Runtime.SetErr(oldErr); } }
public FSCmdExecutor(string namenode, FsShell shell) { this.namenode = namenode; this.shell = shell; }
public _PrivilegedExceptionAction_153(FsShell fsShell, string[] cmdOpts) { this.fsShell = fsShell; this.cmdOpts = cmdOpts; }