/// <exception cref="System.IO.IOException"/>
public static void TrashNonDefaultFS(Configuration conf)
{
conf.SetLong(FsTrashIntervalKey, 10);
{
// 10 minute
// attempt non-default FileSystem trash
FileSystem lfs = FileSystem.GetLocal(conf);
Path p = TestDir;
Path f = new Path(p, "foo/bar");
if (lfs.Exists(p))
{
lfs.Delete(p, true);
}
try
{
FileSystemTestHelper.WriteFile(lfs, f, 10);
FileSystem.CloseAll();
FileSystem localFs = FileSystem.Get(URI.Create("file:///"), conf);
Trash lTrash = new Trash(localFs, conf);
lTrash.MoveToTrash(f.GetParent());
CheckTrash(localFs, lTrash.GetCurrentTrashDir(), f);
}
finally
{
if (lfs.Exists(p))
{
lfs.Delete(p, true);
}
}
}
}
/// <exception cref="System.Exception"/>
internal virtual void VerifyRename(Path srcPath, Path dstPath, bool dstIsDir)
{
localFs.Delete(srcPath, true);
localFs.Delete(dstPath, true);
Path realDstPath = dstPath;
if (dstIsDir)
{
localFs.Mkdirs(dstPath);
realDstPath = new Path(dstPath, srcPath.GetName());
}
// ensure file + checksum are moved
FileSystemTestHelper.WriteFile(localFs, srcPath, 1);
Assert.True(localFs.Exists(localFs.GetChecksumFile(srcPath)));
Assert.True(localFs.Rename(srcPath, dstPath));
Assert.True(localFs.Exists(localFs.GetChecksumFile(realDstPath)
));
// create a file with no checksum, rename, ensure dst checksum is removed
FileSystemTestHelper.WriteFile(localFs.GetRawFileSystem(), srcPath, 1);
NUnit.Framework.Assert.IsFalse(localFs.Exists(localFs.GetChecksumFile(srcPath)));
Assert.True(localFs.Rename(srcPath, dstPath));
NUnit.Framework.Assert.IsFalse(localFs.Exists(localFs.GetChecksumFile(realDstPath
)));
// create file with checksum, rename over prior dst with no checksum
FileSystemTestHelper.WriteFile(localFs, srcPath, 1);
Assert.True(localFs.Exists(localFs.GetChecksumFile(srcPath)));
Assert.True(localFs.Rename(srcPath, dstPath));
Assert.True(localFs.Exists(localFs.GetChecksumFile(realDstPath)
));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestCopy()
{
Path src = new Path(TestRootDir, "dingo");
Path dst = new Path(TestRootDir, "yak");
FileSystemTestHelper.WriteFile(fileSys, src, 1);
Assert.True(FileUtil.Copy(fileSys, src, fileSys, dst, true, false
, conf));
Assert.True(!fileSys.Exists(src) && fileSys.Exists(dst));
Assert.True(FileUtil.Copy(fileSys, dst, fileSys, src, false, false
, conf));
Assert.True(fileSys.Exists(src) && fileSys.Exists(dst));
Assert.True(FileUtil.Copy(fileSys, src, fileSys, dst, true, true
, conf));
Assert.True(!fileSys.Exists(src) && fileSys.Exists(dst));
fileSys.Mkdirs(src);
Assert.True(FileUtil.Copy(fileSys, dst, fileSys, src, false, false
, conf));
Path tmp = new Path(src, dst.GetName());
Assert.True(fileSys.Exists(tmp) && fileSys.Exists(dst));
Assert.True(FileUtil.Copy(fileSys, dst, fileSys, src, false, true
, conf));
Assert.True(fileSys.Delete(tmp, true));
fileSys.Mkdirs(tmp);
try
{
FileUtil.Copy(fileSys, dst, fileSys, src, true, true, conf);
NUnit.Framework.Assert.Fail("Failed to detect existing dir");
}
catch (IOException)
{
}
}
public virtual void TestTruncate()
{
short repl = 3;
int blockSize = 1024;
int numOfBlocks = 2;
DistributedFileSystem fs = cluster.GetFileSystem();
Path dir = GetTestRootPath(fc, "test/hadoop");
Path file = GetTestRootPath(fc, "test/hadoop/file");
byte[] data = FileSystemTestHelper.GetFileData(numOfBlocks, blockSize);
FileSystemTestHelper.CreateFile(fs, file, data, blockSize, repl);
int newLength = blockSize;
bool isReady = fc.Truncate(file, newLength);
NUnit.Framework.Assert.IsTrue("Recovery is not expected.", isReady);
FileStatus fileStatus = fc.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), newLength);
AppendTestUtil.CheckFullFile(fs, file, newLength, data, file.ToString());
ContentSummary cs = fs.GetContentSummary(dir);
NUnit.Framework.Assert.AreEqual("Bad disk space usage", cs.GetSpaceConsumed(), newLength
* repl);
NUnit.Framework.Assert.IsTrue(fs.Delete(dir, true));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestPathEscapes()
{
Path path = new Path(TestRootDir, "foo%bar");
FileSystemTestHelper.WriteFile(fileSys, path, 1);
FileStatus status = fileSys.GetFileStatus(path);
Assert.Equal(path.MakeQualified(fileSys), status.GetPath());
CleanupFile(fileSys, path);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestHasFileDescriptor()
{
Path path = new Path(TestRootDir, "test-file");
FileSystemTestHelper.WriteFile(fileSys, path, 1);
BufferedFSInputStream bis = new BufferedFSInputStream(new RawLocalFileSystem.LocalFSFileInputStream
(this, path), 1024);
NUnit.Framework.Assert.IsNotNull(bis.GetFileDescriptor());
bis.Close();
}
public virtual void TestStripFragmentFromPath()
{
FileSystem fs = FileSystem.GetLocal(new Configuration());
Path pathQualified = TestPath.MakeQualified(fs.GetUri(), fs.GetWorkingDirectory()
);
Path pathWithFragment = new Path(new URI(pathQualified.ToString() + "#glacier"));
// Create test file with fragment
FileSystemTestHelper.CreateFile(fs, pathWithFragment);
Path resolved = fs.ResolvePath(pathWithFragment);
Assert.Equal("resolvePath did not strip fragment from Path", pathQualified
, resolved);
}
/// <summary>Test the capability of setting the working directory.</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestWorkingDirectory()
{
Path origDir = fileSys.GetWorkingDirectory();
Path subdir = new Path(TestRootDir, "new");
try
{
// make sure it doesn't already exist
Assert.True(!fileSys.Exists(subdir));
// make it and check for it
Assert.True(fileSys.Mkdirs(subdir));
Assert.True(fileSys.IsDirectory(subdir));
fileSys.SetWorkingDirectory(subdir);
// create a directory and check for it
Path dir1 = new Path("dir1");
Assert.True(fileSys.Mkdirs(dir1));
Assert.True(fileSys.IsDirectory(dir1));
// delete the directory and make sure it went away
fileSys.Delete(dir1, true);
Assert.True(!fileSys.Exists(dir1));
// create files and manipulate them.
Path file1 = new Path("file1");
Path file2 = new Path("sub/file2");
string contents = FileSystemTestHelper.WriteFile(fileSys, file1, 1);
fileSys.CopyFromLocalFile(file1, file2);
Assert.True(fileSys.Exists(file1));
Assert.True(fileSys.IsFile(file1));
CleanupFile(fileSys, file2);
fileSys.CopyToLocalFile(file1, file2);
CleanupFile(fileSys, file2);
// try a rename
fileSys.Rename(file1, file2);
Assert.True(!fileSys.Exists(file1));
Assert.True(fileSys.Exists(file2));
fileSys.Rename(file2, file1);
// try reading a file
InputStream stm = fileSys.Open(file1);
byte[] buffer = new byte[3];
int bytesRead = stm.Read(buffer, 0, 3);
Assert.Equal(contents, Runtime.GetStringForBytes(buffer
, 0, bytesRead));
stm.Close();
}
finally
{
fileSys.SetWorkingDirectory(origDir);
}
}
// Expected
/// <summary>Test deleting a file, directory, and non-existent path</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestBasicDelete()
{
Path dir1 = new Path(TestRootDir, "dir1");
Path file1 = new Path(TestRootDir, "file1");
Path file2 = new Path(TestRootDir + "/dir1", "file2");
Path file3 = new Path(TestRootDir, "does-not-exist");
Assert.True(fileSys.Mkdirs(dir1));
FileSystemTestHelper.WriteFile(fileSys, file1, 1);
FileSystemTestHelper.WriteFile(fileSys, file2, 1);
NUnit.Framework.Assert.IsFalse("Returned true deleting non-existant path", fileSys
.Delete(file3));
Assert.True("Did not delete file", fileSys.Delete(file1));
Assert.True("Did not delete non-empty dir", fileSys.Delete(dir1
));
}
public virtual void TestRenameReplaceExistingEmptyDirectory()
{
Path src = new Path(TestRootDir, "dir1");
Path dst = new Path(TestRootDir, "dir2");
fileSys.Delete(src, true);
fileSys.Delete(dst, true);
Assert.True(fileSys.Mkdirs(src));
FileSystemTestHelper.WriteFile(fileSys, new Path(src, "file1"), 1);
FileSystemTestHelper.WriteFile(fileSys, new Path(src, "file2"), 1);
Assert.True(fileSys.Mkdirs(dst));
Assert.True(fileSys.Rename(src, dst));
Assert.True(fileSys.Exists(dst));
Assert.True(fileSys.Exists(new Path(dst, "file1")));
Assert.True(fileSys.Exists(new Path(dst, "file2")));
NUnit.Framework.Assert.IsFalse(fileSys.Exists(src));
}
public virtual void TestVerifyChecksum()
{
Path testPath = new Path(TestRootDir, "testPath");
Path testPath11 = new Path(TestRootDir, "testPath11");
FSDataOutputStream fout = localFs.Create(testPath);
fout.Write(Runtime.GetBytesForString("testing"));
fout.Close();
fout = localFs.Create(testPath11);
fout.Write(Runtime.GetBytesForString("testing you"));
fout.Close();
// Exercise some boundary cases - a divisor of the chunk size
// the chunk size, 2x chunk size, and +/-1 around these.
FileSystemTestHelper.ReadFile(localFs, testPath, 128);
FileSystemTestHelper.ReadFile(localFs, testPath, 511);
FileSystemTestHelper.ReadFile(localFs, testPath, 512);
FileSystemTestHelper.ReadFile(localFs, testPath, 513);
FileSystemTestHelper.ReadFile(localFs, testPath, 1023);
FileSystemTestHelper.ReadFile(localFs, testPath, 1024);
FileSystemTestHelper.ReadFile(localFs, testPath, 1025);
localFs.Delete(localFs.GetChecksumFile(testPath), true);
Assert.True("checksum deleted", !localFs.Exists(localFs.GetChecksumFile
(testPath)));
//copying the wrong checksum file
FileUtil.Copy(localFs, localFs.GetChecksumFile(testPath11), localFs, localFs.GetChecksumFile
(testPath), false, true, localFs.GetConf());
Assert.True("checksum exists", localFs.Exists(localFs.GetChecksumFile
(testPath)));
bool errorRead = false;
try
{
FileSystemTestHelper.ReadFile(localFs, testPath, 1024);
}
catch (ChecksumException)
{
errorRead = true;
}
Assert.True("error reading", errorRead);
//now setting verify false, the read should succeed
localFs.SetVerifyChecksum(false);
string str = FileSystemTestHelper.ReadFile(localFs, testPath, 1024).ToString();
Assert.True("read", "testing".Equals(str));
}
/// <exception cref="System.Exception"/>
public virtual void TestSetTimes()
{
Path path = new Path(TestRootDir, "set-times");
FileSystemTestHelper.WriteFile(fileSys, path, 1);
// test only to the nearest second, as the raw FS may not
// support millisecond timestamps
long newModTime = 12345000;
FileStatus status = fileSys.GetFileStatus(path);
Assert.True("check we're actually changing something", newModTime
!= status.GetModificationTime());
long accessTime = status.GetAccessTime();
fileSys.SetTimes(path, newModTime, -1);
status = fileSys.GetFileStatus(path);
Assert.Equal(newModTime, status.GetModificationTime());
Assert.Equal(accessTime, status.GetAccessTime());
}
public virtual void TestMultiChunkFile()
{
Path testPath = new Path(TestRootDir, "testMultiChunk");
FSDataOutputStream fout = localFs.Create(testPath);
for (int i = 0; i < 1000; i++)
{
fout.Write(Runtime.GetBytesForString(("testing" + i)));
}
fout.Close();
// Exercise some boundary cases - a divisor of the chunk size
// the chunk size, 2x chunk size, and +/-1 around these.
FileSystemTestHelper.ReadFile(localFs, testPath, 128);
FileSystemTestHelper.ReadFile(localFs, testPath, 511);
FileSystemTestHelper.ReadFile(localFs, testPath, 512);
FileSystemTestHelper.ReadFile(localFs, testPath, 513);
FileSystemTestHelper.ReadFile(localFs, testPath, 1023);
FileSystemTestHelper.ReadFile(localFs, testPath, 1024);
FileSystemTestHelper.ReadFile(localFs, testPath, 1025);
}
public virtual void TestCorruptedChecksum()
{
Path testPath = new Path(TestRootDir, "testCorruptChecksum");
Path checksumPath = localFs.GetChecksumFile(testPath);
// write a file to generate checksum
FSDataOutputStream @out = localFs.Create(testPath, true);
@out.Write(Runtime.GetBytesForString("testing 1 2 3"));
@out.Close();
Assert.True(localFs.Exists(checksumPath));
FileStatus stat = localFs.GetFileStatus(checksumPath);
// alter file directly so checksum is invalid
@out = localFs.GetRawFileSystem().Create(testPath, true);
@out.Write(Runtime.GetBytesForString("testing stale checksum"));
@out.Close();
Assert.True(localFs.Exists(checksumPath));
// checksum didn't change on disk
Assert.Equal(stat, localFs.GetFileStatus(checksumPath));
Exception e = null;
try
{
localFs.SetVerifyChecksum(true);
FileSystemTestHelper.ReadFile(localFs, testPath, 1024);
}
catch (ChecksumException ce)
{
e = ce;
}
finally
{
NUnit.Framework.Assert.IsNotNull("got checksum error", e);
}
localFs.SetVerifyChecksum(false);
string str = FileSystemTestHelper.ReadFile(localFs, testPath, 1024);
Assert.Equal("testing stale checksum", str);
}
public virtual void TestTruncatedChecksum()
{
Path testPath = new Path(TestRootDir, "testtruncatedcrc");
FSDataOutputStream fout = localFs.Create(testPath);
fout.Write(Runtime.GetBytesForString("testing truncation"));
fout.Close();
// Read in the checksum
Path checksumFile = localFs.GetChecksumFile(testPath);
FileSystem rawFs = localFs.GetRawFileSystem();
FSDataInputStream checksumStream = rawFs.Open(checksumFile);
byte[] buf = new byte[8192];
int read = checksumStream.Read(buf, 0, buf.Length);
checksumStream.Close();
// Now rewrite the checksum file with the last byte missing
FSDataOutputStream replaceStream = rawFs.Create(checksumFile);
replaceStream.Write(buf, 0, read - 1);
replaceStream.Close();
// Now reading the file should fail with a ChecksumException
try
{
FileSystemTestHelper.ReadFile(localFs, testPath, 1024);
NUnit.Framework.Assert.Fail("Did not throw a ChecksumException when reading truncated "
+ "crc file");
}
catch (ChecksumException)
{
}
// telling it not to verify checksums, should avoid issue.
localFs.SetVerifyChecksum(false);
string str = FileSystemTestHelper.ReadFile(localFs, testPath, 1024).ToString();
Assert.True("read", "testing truncation".Equals(str));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestCreateFileAndMkdirs()
{
Path test_dir = new Path(TestRootDir, "test_dir");
Path test_file = new Path(test_dir, "file1");
Assert.True(fileSys.Mkdirs(test_dir));
int fileSize = new Random().Next(1 << 20) + 1;
FileSystemTestHelper.WriteFile(fileSys, test_file, fileSize);
{
//check FileStatus and ContentSummary
FileStatus status = fileSys.GetFileStatus(test_file);
Assert.Equal(fileSize, status.GetLen());
ContentSummary summary = fileSys.GetContentSummary(test_dir);
Assert.Equal(fileSize, summary.GetLength());
}
// creating dir over a file
Path bad_dir = new Path(test_file, "another_dir");
try
{
fileSys.Mkdirs(bad_dir);
NUnit.Framework.Assert.Fail("Failed to detect existing file in path");
}
catch (ParentNotDirectoryException)
{
}
// Expected
try
{
fileSys.Mkdirs(null);
NUnit.Framework.Assert.Fail("Failed to detect null in mkdir arg");
}
catch (ArgumentException)
{
}
}
/// <exception cref="System.Exception"/>
public virtual void TestTrashEmptier()
{
Configuration conf = new Configuration();
// Trash with 12 second deletes and 6 seconds checkpoints
conf.Set(FsTrashIntervalKey, "0.2");
// 12 seconds
conf.SetClass("fs.file.impl", typeof(TestTrash.TestLFS), typeof(FileSystem));
conf.Set(FsTrashCheckpointIntervalKey, "0.1");
// 6 seconds
FileSystem fs = FileSystem.GetLocal(conf);
conf.Set("fs.default.name", fs.GetUri().ToString());
Trash trash = new Trash(conf);
// Start Emptier in background
Runnable emptier = trash.GetEmptier();
Thread emptierThread = new Thread(emptier);
emptierThread.Start();
FsShell shell = new FsShell();
shell.SetConf(conf);
shell.Init();
// First create a new directory with mkdirs
Path myPath = new Path(TestDir, "test/mkdirs");
Mkdir(fs, myPath);
int fileIndex = 0;
ICollection <string> checkpoints = new HashSet <string>();
while (true)
{
// Create a file with a new name
Path myFile = new Path(TestDir, "test/mkdirs/myFile" + fileIndex++);
FileSystemTestHelper.WriteFile(fs, myFile, 10);
// Delete the file to trash
string[] args = new string[2];
args[0] = "-rm";
args[1] = myFile.ToString();
int val = -1;
try
{
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
Assert.True(val == 0);
Path trashDir = shell.GetCurrentTrashDir();
FileStatus[] files = fs.ListStatus(trashDir.GetParent());
// Scan files in .Trash and add them to set of checkpoints
foreach (FileStatus file in files)
{
string fileName = file.GetPath().GetName();
checkpoints.AddItem(fileName);
}
// If checkpoints has 4 objects it is Current + 3 checkpoint directories
if (checkpoints.Count == 4)
{
// The actual contents should be smaller since the last checkpoint
// should've been deleted and Current might not have been recreated yet
Assert.True(checkpoints.Count > files.Length);
break;
}
Thread.Sleep(5000);
}
emptierThread.Interrupt();
emptierThread.Join();
}
/// <summary>
/// Test trash for the shell's delete command for the default file system
/// specified in the paramter conf
/// </summary>
/// <param name="conf"></param>
/// <param name="base">- the base path where files are created</param>
/// <param name="trashRoot">- the expected place where the trashbin resides</param>
/// <exception cref="System.IO.IOException"/>
public static void TrashShell(Configuration conf, Path @base, FileSystem trashRootFs
, Path trashRoot)
{
FileSystem fs = FileSystem.Get(conf);
conf.SetLong(FsTrashIntervalKey, 0);
// disabled
NUnit.Framework.Assert.IsFalse(new Trash(conf).IsEnabled());
conf.SetLong(FsTrashIntervalKey, 10);
// 10 minute
Assert.True(new Trash(conf).IsEnabled());
FsShell shell = new FsShell();
shell.SetConf(conf);
if (trashRoot == null)
{
trashRoot = shell.GetCurrentTrashDir();
}
if (trashRootFs == null)
{
trashRootFs = fs;
}
// First create a new directory with mkdirs
Path myPath = new Path(@base, "test/mkdirs");
Mkdir(fs, myPath);
// Second, create a file in that directory.
Path myFile = new Path(@base, "test/mkdirs/myFile");
FileSystemTestHelper.WriteFile(fs, myFile, 10);
{
// Verify that expunge without Trash directory
// won't throw Exception
string[] args = new string[1];
args[0] = "-expunge";
int val = -1;
try
{
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
Assert.True(val == 0);
}
{
// Verify that we succeed in removing the file we created.
// This should go into Trash.
string[] args = new string[2];
args[0] = "-rm";
args[1] = myFile.ToString();
int val = -1;
try
{
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
Assert.True(val == 0);
CheckTrash(trashRootFs, trashRoot, fs.MakeQualified(myFile));
}
// Verify that we can recreate the file
FileSystemTestHelper.WriteFile(fs, myFile, 10);
{
// Verify that we succeed in removing the file we re-created
string[] args = new string[2];
args[0] = "-rm";
args[1] = new Path(@base, "test/mkdirs/myFile").ToString();
int val = -1;
try
{
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
Assert.True(val == 0);
}
// Verify that we can recreate the file
FileSystemTestHelper.WriteFile(fs, myFile, 10);
{
// Verify that we succeed in removing the whole directory
// along with the file inside it.
string[] args = new string[2];
args[0] = "-rmr";
args[1] = new Path(@base, "test/mkdirs").ToString();
int val = -1;
try
{
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
Assert.True(val == 0);
}
// recreate directory
Mkdir(fs, myPath);
{
// Verify that we succeed in removing the whole directory
string[] args = new string[2];
args[0] = "-rmr";
args[1] = new Path(@base, "test/mkdirs").ToString();
int val = -1;
try
{
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
Assert.True(val == 0);
}
{
// Check that we can delete a file from the trash
Path toErase = new Path(trashRoot, "toErase");
int retVal = -1;
FileSystemTestHelper.WriteFile(trashRootFs, toErase, 10);
try
{
retVal = shell.Run(new string[] { "-rm", toErase.ToString() });
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
Assert.True(retVal == 0);
CheckNotInTrash(trashRootFs, trashRoot, toErase.ToString());
CheckNotInTrash(trashRootFs, trashRoot, toErase.ToString() + ".1");
}
{
// simulate Trash removal
string[] args = new string[1];
args[0] = "-expunge";
int val = -1;
try
{
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
Assert.True(val == 0);
}
// verify that after expunging the Trash, it really goes away
CheckNotInTrash(trashRootFs, trashRoot, new Path(@base, "test/mkdirs/myFile").ToString
());
// recreate directory and file
Mkdir(fs, myPath);
FileSystemTestHelper.WriteFile(fs, myFile, 10);
{
// remove file first, then remove directory
string[] args = new string[2];
args[0] = "-rm";
args[1] = myFile.ToString();
int val = -1;
try
{
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
Assert.True(val == 0);
CheckTrash(trashRootFs, trashRoot, myFile);
args = new string[2];
args[0] = "-rmr";
args[1] = myPath.ToString();
val = -1;
try
{
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
Assert.True(val == 0);
CheckTrash(trashRootFs, trashRoot, myPath);
}
{
// attempt to remove parent of trash
string[] args = new string[2];
args[0] = "-rmr";
args[1] = trashRoot.GetParent().GetParent().ToString();
int val = -1;
try
{
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
Assert.Equal("exit code", 1, val);
Assert.True(trashRootFs.Exists(trashRoot));
}
// Verify skip trash option really works
// recreate directory and file
Mkdir(fs, myPath);
FileSystemTestHelper.WriteFile(fs, myFile, 10);
{
// Verify that skip trash option really skips the trash for files (rm)
string[] args = new string[3];
args[0] = "-rm";
args[1] = "-skipTrash";
args[2] = myFile.ToString();
int val = -1;
try
{
// Clear out trash
Assert.Equal("-expunge failed", 0, shell.Run(new string[] { "-expunge" }));
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
NUnit.Framework.Assert.IsFalse("Expected TrashRoot (" + trashRoot + ") to exist in file system:"
+ trashRootFs.GetUri(), trashRootFs.Exists(trashRoot));
// No new Current should be created
NUnit.Framework.Assert.IsFalse(fs.Exists(myFile));
Assert.True(val == 0);
}
// recreate directory and file
Mkdir(fs, myPath);
FileSystemTestHelper.WriteFile(fs, myFile, 10);
{
// Verify that skip trash option really skips the trash for rmr
string[] args = new string[3];
args[0] = "-rmr";
args[1] = "-skipTrash";
args[2] = myPath.ToString();
int val = -1;
try
{
// Clear out trash
Assert.Equal(0, shell.Run(new string[] { "-expunge" }));
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
NUnit.Framework.Assert.IsFalse(trashRootFs.Exists(trashRoot));
// No new Current should be created
NUnit.Framework.Assert.IsFalse(fs.Exists(myPath));
NUnit.Framework.Assert.IsFalse(fs.Exists(myFile));
Assert.True(val == 0);
}
{
// deleting same file multiple times
int val = -1;
Mkdir(fs, myPath);
try
{
Assert.Equal(0, shell.Run(new string[] { "-expunge" }));
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from fs expunge " + e.GetLocalizedMessage
());
}
// create a file in that directory.
myFile = new Path(@base, "test/mkdirs/myFile");
string[] args = new string[] { "-rm", myFile.ToString() };
int num_runs = 10;
for (int i = 0; i < num_runs; i++)
{
//create file
FileSystemTestHelper.WriteFile(fs, myFile, 10);
// delete file
try
{
val = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
Assert.True(val == 0);
}
// current trash directory
Path trashDir = Path.MergePaths(new Path(trashRoot.ToUri().GetPath()), new Path(myFile
.GetParent().ToUri().GetPath()));
System.Console.Out.WriteLine("Deleting same myFile: myFile.parent=" + myFile.GetParent
().ToUri().GetPath() + "; trashroot=" + trashRoot.ToUri().GetPath() + "; trashDir="
+ trashDir.ToUri().GetPath());
int count = CountSameDeletedFiles(fs, trashDir, myFile);
System.Console.Out.WriteLine("counted " + count + " files " + myFile.GetName() +
"* in " + trashDir);
Assert.True(count == num_runs);
}
{
//Verify skipTrash option is suggested when rm fails due to its absence
string[] args = new string[2];
args[0] = "-rmr";
args[1] = "/";
//This always contains trash directory
TextWriter stdout = System.Console.Out;
TextWriter stderr = System.Console.Error;
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
TextWriter newOut = new TextWriter(byteStream);
Runtime.SetOut(newOut);
Runtime.SetErr(newOut);
try
{
shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
}
string output = byteStream.ToString();
Runtime.SetOut(stdout);
Runtime.SetErr(stderr);
Assert.True("skipTrash wasn't suggested as remedy to failed rm command"
+ " or we deleted / even though we could not get server defaults", output.IndexOf
("Consider using -skipTrash option") != -1 || output.IndexOf("Failed to determine server trash configuration"
) != -1);
}
{
// Verify old checkpoint format is recognized
// emulate two old trash checkpoint directories, one that is old enough
// to be deleted on the next expunge and one that isn't.
long trashInterval = conf.GetLong(FsTrashIntervalKey, FsTrashIntervalDefault);
long now = Time.Now();
DateFormat oldCheckpointFormat = new SimpleDateFormat("yyMMddHHmm");
Path dirToDelete = new Path(trashRoot.GetParent(), oldCheckpointFormat.Format(now
- (trashInterval * 60 * 1000) - 1));
Path dirToKeep = new Path(trashRoot.GetParent(), oldCheckpointFormat.Format(now));
Mkdir(trashRootFs, dirToDelete);
Mkdir(trashRootFs, dirToKeep);
// Clear out trash
int rc = -1;
try
{
rc = shell.Run(new string[] { "-expunge" });
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from fs expunge " + e.GetLocalizedMessage
());
}
Assert.Equal(0, rc);
NUnit.Framework.Assert.IsFalse("old checkpoint format not recognized", trashRootFs
.Exists(dirToDelete));
Assert.True("old checkpoint format directory should not be removed"
, trashRootFs.Exists(dirToKeep));
}
}
/// <summary>
/// test same file deletion - multiple time
/// this is more of a performance test - shouldn't be run as a unit test
/// </summary>
/// <exception cref="System.IO.IOException"/>
public static void PerformanceTestDeleteSameFile()
{
Path @base = TestDir;
Configuration conf = new Configuration();
conf.SetClass("fs.file.impl", typeof(TestTrash.TestLFS), typeof(FileSystem));
FileSystem fs = FileSystem.GetLocal(conf);
conf.Set("fs.defaultFS", fs.GetUri().ToString());
conf.SetLong(FsTrashIntervalKey, 10);
//minutes..
FsShell shell = new FsShell();
shell.SetConf(conf);
//Path trashRoot = null;
Path myPath = new Path(@base, "test/mkdirs");
Mkdir(fs, myPath);
// create a file in that directory.
Path myFile;
long start;
long first = 0;
int retVal = 0;
int factor = 10;
// how much slower any of subsequent deletion can be
myFile = new Path(@base, "test/mkdirs/myFile");
string[] args = new string[] { "-rm", myFile.ToString() };
int iters = 1000;
for (int i = 0; i < iters; i++)
{
FileSystemTestHelper.WriteFile(fs, myFile, 10);
start = Time.Now();
try
{
retVal = shell.Run(args);
}
catch (Exception e)
{
System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage
());
throw new IOException(e.Message);
}
Assert.True(retVal == 0);
long iterTime = Time.Now() - start;
// take median of the first 10 runs
if (i < 10)
{
if (i == 0)
{
first = iterTime;
}
else
{
first = (first + iterTime) / 2;
}
}
// we don't want to print every iteration - let's do every 10th
int print_freq = iters / 10;
if (i > 10)
{
if ((i % print_freq) == 0)
{
System.Console.Out.WriteLine("iteration=" + i + ";res =" + retVal + "; start=" +
start + "; iterTime = " + iterTime + " vs. firstTime=" + first);
}
long factoredTime = first * factor;
Assert.True(iterTime < factoredTime);
}
}
}
