/// <exception cref="System.Exception"/>
public virtual void TestSetrepIncWithUnderReplicatedBlocks()
{
// 1 min timeout
Configuration conf = new HdfsConfiguration();
short ReplicationFactor = 2;
string FileName = "/testFile";
Path FilePath = new Path(FileName);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(ReplicationFactor
+ 1).Build();
try
{
// create a file with one block with a replication factor of 2
FileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, FilePath, 1L, ReplicationFactor, 1L);
DFSTestUtil.WaitReplication(fs, FilePath, ReplicationFactor);
// remove one replica from the blocksMap so block becomes under-replicated
// but the block does not get put into the under-replicated blocks queue
BlockManager bm = cluster.GetNamesystem().GetBlockManager();
ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, FilePath);
DatanodeDescriptor dn = bm.blocksMap.GetStorages(b.GetLocalBlock()).GetEnumerator
().Next().GetDatanodeDescriptor();
bm.AddToInvalidates(b.GetLocalBlock(), dn);
Sharpen.Thread.Sleep(5000);
bm.blocksMap.RemoveNode(b.GetLocalBlock(), dn);
// increment this file's replication factor
FsShell shell = new FsShell(conf);
NUnit.Framework.Assert.AreEqual(0, shell.Run(new string[] { "-setrep", "-w", Sharpen.Extensions.ToString
(1 + ReplicationFactor), FileName }));
}
finally
{
cluster.Shutdown();
}
}
private void Execute(string[] args, string namenode)
{
FsShell shell = new FsShell();
FileSystem fs = null;
try
{
ToolRunner.Run(shell, args);
fs = FileSystem.Get(NameNode.GetUri(NameNode.GetAddress(namenode)), shell.GetConf
());
NUnit.Framework.Assert.IsTrue("Directory does not get created", fs.IsDirectory(new
Path("/data")));
fs.Delete(new Path("/data"), true);
}
catch (Exception e)
{
System.Console.Error.WriteLine(e.Message);
Sharpen.Runtime.PrintStackTrace(e);
}
finally
{
if (fs != null)
{
try
{
fs.Close();
}
catch (IOException)
{
}
}
}
}
public static void Setup()
{
conf.SetClass("fs.file.impl", typeof(TestFsShellReturnCode.LocalFileSystemExtn),
typeof(LocalFileSystem));
fileSys = FileSystem.Get(conf);
fsShell = new FsShell(conf);
}
/// <exception cref="System.Exception"/>
public virtual void TestRmWithNonexistentGlob()
{
Configuration conf = new Configuration();
FsShell shell = new FsShell();
shell.SetConf(conf);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
TextWriter err = new TextWriter(bytes);
TextWriter oldErr = System.Console.Error;
Runtime.SetErr(err);
string results;
try
{
int exit = shell.Run(new string[] { "-rm", "nomatch*" });
Assert.Equal(1, exit);
results = bytes.ToString();
Assert.True(results.Contains("rm: `nomatch*': No such file or directory"
));
}
finally
{
IOUtils.CloseStream(err);
Runtime.SetErr(oldErr);
}
}
/// <exception cref="System.IO.IOException"/>
internal static void Setrep(int fromREP, int toREP, bool simulatedStorage)
{
Configuration conf = new HdfsConfiguration();
if (simulatedStorage)
{
SimulatedFSDataset.SetFactory(conf);
}
conf.Set(DFSConfigKeys.DfsReplicationKey, string.Empty + fromREP);
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L);
conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString
(2));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(10).Build(
);
FileSystem fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem
);
try
{
Path root = TestDFSShell.Mkdir(fs, new Path("/test/setrep" + fromREP + "-" + toREP
));
Path f = TestDFSShell.WriteFile(fs, new Path(root, "foo"));
{
// Verify setrep for changing replication
string[] args = new string[] { "-setrep", "-w", string.Empty + toREP, string.Empty
+ f };
FsShell shell = new FsShell();
shell.SetConf(conf);
try
{
NUnit.Framework.Assert.AreEqual(0, shell.Run(args));
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue("-setrep " + e, false);
}
}
//get fs again since the old one may be closed
fs = cluster.GetFileSystem();
FileStatus file = fs.GetFileStatus(f);
long len = file.GetLen();
foreach (BlockLocation locations in fs.GetFileBlockLocations(file, 0, len))
{
NUnit.Framework.Assert.IsTrue(locations.GetHosts().Length == toREP);
}
TestDFSShell.Show("done setrep waiting: " + root);
}
finally
{
try
{
fs.Close();
}
catch (Exception)
{
}
cluster.Shutdown();
}
}
public virtual void TestRenameSnapshotCommandWithIllegalArguments()
{
ByteArrayOutputStream @out = new ByteArrayOutputStream();
TextWriter psOut = new TextWriter(@out);
Runtime.SetOut(psOut);
Runtime.SetErr(psOut);
FsShell shell = new FsShell();
shell.SetConf(conf);
string[] argv1 = new string[] { "-renameSnapshot", "/tmp", "s1" };
int val = shell.Run(argv1);
NUnit.Framework.Assert.IsTrue(val == -1);
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains(argv1[0] + ": Incorrect number of arguments."
));
@out.Reset();
string[] argv2 = new string[] { "-renameSnapshot", "/tmp", "s1", "s2", "s3" };
val = shell.Run(argv2);
NUnit.Framework.Assert.IsTrue(val == -1);
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains(argv2[0] + ": Incorrect number of arguments."
));
psOut.Close();
@out.Close();
}
/// <summary>
/// Adding as part of jira HDFS-5343
/// Test for checking the cat command on snapshot path it
/// cannot read a file beyond snapshot file length
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotFileLengthWithCatCommand()
{
FSDataInputStream fis = null;
FileStatus fileStatus = null;
int bytesRead;
byte[] buffer = new byte[Blocksize * 8];
hdfs.Mkdirs(sub);
Path file1 = new Path(sub, file1Name);
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, Seed);
hdfs.AllowSnapshot(sub);
hdfs.CreateSnapshot(sub, snapshot1);
DFSTestUtil.AppendFile(hdfs, file1, Blocksize);
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.GetFileStatus(file1);
NUnit.Framework.Assert.AreEqual("Unexpected file length", Blocksize * 2, fileStatus
.GetLen());
fis = hdfs.Open(file1);
bytesRead = fis.Read(buffer, 0, buffer.Length);
NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize * 2, bytesRead
);
fis.Close();
Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name);
fis = hdfs.Open(file1snap1);
fileStatus = hdfs.GetFileStatus(file1snap1);
NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), Blocksize);
// Make sure we can only read up to the snapshot length.
bytesRead = fis.Read(buffer, 0, buffer.Length);
NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize, bytesRead);
fis.Close();
TextWriter outBackup = System.Console.Out;
TextWriter errBackup = System.Console.Error;
ByteArrayOutputStream bao = new ByteArrayOutputStream();
Runtime.SetOut(new TextWriter(bao));
Runtime.SetErr(new TextWriter(bao));
// Make sure we can cat the file upto to snapshot length
FsShell shell = new FsShell();
try
{
ToolRunner.Run(conf, shell, new string[] { "-cat", "/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1" });
NUnit.Framework.Assert.AreEqual("Unexpected # bytes from -cat", Blocksize, bao.Size
());
}
finally
{
Runtime.SetOut(outBackup);
Runtime.SetErr(errBackup);
}
}
/// <exception cref="System.Exception"/>
internal static string ExecCmd(FsShell shell, string[] args)
{
ByteArrayOutputStream baout = new ByteArrayOutputStream();
TextWriter @out = new TextWriter(baout, true);
TextWriter old = System.Console.Out;
Runtime.SetOut(@out);
int ret = shell.Run(args);
@out.Close();
Runtime.SetOut(old);
return(ret.ToString());
}
/// <exception cref="System.Exception"/>
public virtual void Execute(Configuration conf, FileSystem fs)
{
fs.Mkdirs(new Path(TestFsShellPermission.TestRoot));
this._enclosing.CreateFiles(fs, TestFsShellPermission.TestRoot, this.fileEntries);
FsShell fsShell = new FsShell(conf);
string deletePath = TestFsShellPermission.TestRoot + "/" + this.deleteEntry.GetPath
();
string[] tmpCmdOpts = StringUtils.Split(this.cmdAndOptions);
AList <string> tmpArray = new AList <string>(Arrays.AsList(tmpCmdOpts));
tmpArray.AddItem(deletePath);
string[] cmdOpts = Sharpen.Collections.ToArray(tmpArray, new string[tmpArray.Count
]);
this.userUgi.DoAs(new _PrivilegedExceptionAction_153(fsShell, cmdOpts));
bool deleted = !fs.Exists(new Path(deletePath));
NUnit.Framework.Assert.AreEqual(this.expectedToDelete, deleted);
TestFsShellPermission.Deldir(fs, TestFsShellPermission.TestRoot);
}
/// <exception cref="System.Exception"/>
public virtual void TestGetWithInvalidSourcePathShouldNotDisplayNullInConsole()
{
Configuration conf = new Configuration();
FsShell shell = new FsShell();
shell.SetConf(conf);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
TextWriter @out = new TextWriter(bytes);
TextWriter oldErr = System.Console.Error;
Runtime.SetErr(@out);
string results;
try
{
Path tdir = new Path(TestRootDir, "notNullCopy");
fileSys.Delete(tdir, true);
fileSys.Mkdirs(tdir);
string[] args = new string[3];
args[0] = "-get";
args[1] = new Path(tdir.ToUri().GetPath(), "/invalidSrc").ToString();
args[2] = new Path(tdir.ToUri().GetPath(), "/invalidDst").ToString();
Assert.True("file exists", !fileSys.Exists(new Path(args[1])));
Assert.True("file exists", !fileSys.Exists(new Path(args[2])));
int run = shell.Run(args);
results = bytes.ToString();
Assert.Equal("Return code should be 1", 1, run);
Assert.True(" Null is coming when source path is invalid. ", !results
.Contains("get: null"));
Assert.True(" Not displaying the intended message ", results.Contains
("get: `" + args[1] + "': No such file or directory"));
}
finally
{
IOUtils.CloseStream(@out);
Runtime.SetErr(oldErr);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestRmForceWithNonexistentGlob()
{
Configuration conf = new Configuration();
FsShell shell = new FsShell();
shell.SetConf(conf);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
TextWriter err = new TextWriter(bytes);
TextWriter oldErr = System.Console.Error;
Runtime.SetErr(err);
try
{
int exit = shell.Run(new string[] { "-rm", "-f", "nomatch*" });
Assert.Equal(0, exit);
Assert.True(bytes.ToString().IsEmpty());
}
finally
{
IOUtils.CloseStream(err);
Runtime.SetErr(oldErr);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidDefaultFS()
{
// if default fs doesn't exist or is invalid, but the path provided in
// arguments is valid - fsshell should work
FsShell shell = new FsShell();
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, "hhhh://doesnotexist/");
shell.SetConf(conf);
string[] args = new string[2];
args[0] = "-ls";
args[1] = "file:///";
// this is valid, so command should run
int res = shell.Run(args);
System.Console.Out.WriteLine("res =" + res);
shell.SetConf(conf);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
TextWriter @out = new TextWriter(bytes);
TextWriter oldErr = System.Console.Error;
Runtime.SetErr(@out);
string results;
try
{
int run = shell.Run(args);
results = bytes.ToString();
Log.Info("result=" + results);
Assert.True("Return code should be 0", run == 0);
}
finally
{
IOUtils.CloseStream(@out);
Runtime.SetErr(oldErr);
}
}
public FSCmdExecutor(string namenode, FsShell shell)
{
this.namenode = namenode;
this.shell = shell;
}
public _PrivilegedExceptionAction_153(FsShell fsShell, string[] cmdOpts)
{
this.fsShell = fsShell;
this.cmdOpts = cmdOpts;
}
