/// <exception cref="System.IO.IOException"/> public virtual void TestCopy() { Path src = new Path(TestRootDir, "dingo"); Path dst = new Path(TestRootDir, "yak"); FileSystemTestHelper.WriteFile(fileSys, src, 1); Assert.True(FileUtil.Copy(fileSys, src, fileSys, dst, true, false , conf)); Assert.True(!fileSys.Exists(src) && fileSys.Exists(dst)); Assert.True(FileUtil.Copy(fileSys, dst, fileSys, src, false, false , conf)); Assert.True(fileSys.Exists(src) && fileSys.Exists(dst)); Assert.True(FileUtil.Copy(fileSys, src, fileSys, dst, true, true , conf)); Assert.True(!fileSys.Exists(src) && fileSys.Exists(dst)); fileSys.Mkdirs(src); Assert.True(FileUtil.Copy(fileSys, dst, fileSys, src, false, false , conf)); Path tmp = new Path(src, dst.GetName()); Assert.True(fileSys.Exists(tmp) && fileSys.Exists(dst)); Assert.True(FileUtil.Copy(fileSys, dst, fileSys, src, false, true , conf)); Assert.True(fileSys.Delete(tmp, true)); fileSys.Mkdirs(tmp); try { FileUtil.Copy(fileSys, dst, fileSys, src, true, true, conf); NUnit.Framework.Assert.Fail("Failed to detect existing dir"); } catch (IOException) { } }
/// <exception cref="System.IO.IOException"/> public static void TrashNonDefaultFS(Configuration conf) { conf.SetLong(FsTrashIntervalKey, 10); { // 10 minute // attempt non-default FileSystem trash FileSystem lfs = FileSystem.GetLocal(conf); Path p = TestDir; Path f = new Path(p, "foo/bar"); if (lfs.Exists(p)) { lfs.Delete(p, true); } try { FileSystemTestHelper.WriteFile(lfs, f, 10); FileSystem.CloseAll(); FileSystem localFs = FileSystem.Get(URI.Create("file:///"), conf); Trash lTrash = new Trash(localFs, conf); lTrash.MoveToTrash(f.GetParent()); CheckTrash(localFs, lTrash.GetCurrentTrashDir(), f); } finally { if (lfs.Exists(p)) { lfs.Delete(p, true); } } } }
/// <exception cref="System.Exception"/> internal virtual void VerifyRename(Path srcPath, Path dstPath, bool dstIsDir) { localFs.Delete(srcPath, true); localFs.Delete(dstPath, true); Path realDstPath = dstPath; if (dstIsDir) { localFs.Mkdirs(dstPath); realDstPath = new Path(dstPath, srcPath.GetName()); } // ensure file + checksum are moved FileSystemTestHelper.WriteFile(localFs, srcPath, 1); Assert.True(localFs.Exists(localFs.GetChecksumFile(srcPath))); Assert.True(localFs.Rename(srcPath, dstPath)); Assert.True(localFs.Exists(localFs.GetChecksumFile(realDstPath) )); // create a file with no checksum, rename, ensure dst checksum is removed FileSystemTestHelper.WriteFile(localFs.GetRawFileSystem(), srcPath, 1); NUnit.Framework.Assert.IsFalse(localFs.Exists(localFs.GetChecksumFile(srcPath))); Assert.True(localFs.Rename(srcPath, dstPath)); NUnit.Framework.Assert.IsFalse(localFs.Exists(localFs.GetChecksumFile(realDstPath ))); // create file with checksum, rename over prior dst with no checksum FileSystemTestHelper.WriteFile(localFs, srcPath, 1); Assert.True(localFs.Exists(localFs.GetChecksumFile(srcPath))); Assert.True(localFs.Rename(srcPath, dstPath)); Assert.True(localFs.Exists(localFs.GetChecksumFile(realDstPath) )); }
/// <exception cref="System.IO.IOException"/> public virtual void TestPathEscapes() { Path path = new Path(TestRootDir, "foo%bar"); FileSystemTestHelper.WriteFile(fileSys, path, 1); FileStatus status = fileSys.GetFileStatus(path); Assert.Equal(path.MakeQualified(fileSys), status.GetPath()); CleanupFile(fileSys, path); }
/// <exception cref="System.IO.IOException"/> public virtual void TestHasFileDescriptor() { Path path = new Path(TestRootDir, "test-file"); FileSystemTestHelper.WriteFile(fileSys, path, 1); BufferedFSInputStream bis = new BufferedFSInputStream(new RawLocalFileSystem.LocalFSFileInputStream (this, path), 1024); NUnit.Framework.Assert.IsNotNull(bis.GetFileDescriptor()); bis.Close(); }
/// <summary>Test the capability of setting the working directory.</summary> /// <exception cref="System.IO.IOException"/> public virtual void TestWorkingDirectory() { Path origDir = fileSys.GetWorkingDirectory(); Path subdir = new Path(TestRootDir, "new"); try { // make sure it doesn't already exist Assert.True(!fileSys.Exists(subdir)); // make it and check for it Assert.True(fileSys.Mkdirs(subdir)); Assert.True(fileSys.IsDirectory(subdir)); fileSys.SetWorkingDirectory(subdir); // create a directory and check for it Path dir1 = new Path("dir1"); Assert.True(fileSys.Mkdirs(dir1)); Assert.True(fileSys.IsDirectory(dir1)); // delete the directory and make sure it went away fileSys.Delete(dir1, true); Assert.True(!fileSys.Exists(dir1)); // create files and manipulate them. Path file1 = new Path("file1"); Path file2 = new Path("sub/file2"); string contents = FileSystemTestHelper.WriteFile(fileSys, file1, 1); fileSys.CopyFromLocalFile(file1, file2); Assert.True(fileSys.Exists(file1)); Assert.True(fileSys.IsFile(file1)); CleanupFile(fileSys, file2); fileSys.CopyToLocalFile(file1, file2); CleanupFile(fileSys, file2); // try a rename fileSys.Rename(file1, file2); Assert.True(!fileSys.Exists(file1)); Assert.True(fileSys.Exists(file2)); fileSys.Rename(file2, file1); // try reading a file InputStream stm = fileSys.Open(file1); byte[] buffer = new byte[3]; int bytesRead = stm.Read(buffer, 0, 3); Assert.Equal(contents, Runtime.GetStringForBytes(buffer , 0, bytesRead)); stm.Close(); } finally { fileSys.SetWorkingDirectory(origDir); } }
// Expected /// <summary>Test deleting a file, directory, and non-existent path</summary> /// <exception cref="System.IO.IOException"/> public virtual void TestBasicDelete() { Path dir1 = new Path(TestRootDir, "dir1"); Path file1 = new Path(TestRootDir, "file1"); Path file2 = new Path(TestRootDir + "/dir1", "file2"); Path file3 = new Path(TestRootDir, "does-not-exist"); Assert.True(fileSys.Mkdirs(dir1)); FileSystemTestHelper.WriteFile(fileSys, file1, 1); FileSystemTestHelper.WriteFile(fileSys, file2, 1); NUnit.Framework.Assert.IsFalse("Returned true deleting non-existant path", fileSys .Delete(file3)); Assert.True("Did not delete file", fileSys.Delete(file1)); Assert.True("Did not delete non-empty dir", fileSys.Delete(dir1 )); }
public virtual void TestRenameReplaceExistingEmptyDirectory() { Path src = new Path(TestRootDir, "dir1"); Path dst = new Path(TestRootDir, "dir2"); fileSys.Delete(src, true); fileSys.Delete(dst, true); Assert.True(fileSys.Mkdirs(src)); FileSystemTestHelper.WriteFile(fileSys, new Path(src, "file1"), 1); FileSystemTestHelper.WriteFile(fileSys, new Path(src, "file2"), 1); Assert.True(fileSys.Mkdirs(dst)); Assert.True(fileSys.Rename(src, dst)); Assert.True(fileSys.Exists(dst)); Assert.True(fileSys.Exists(new Path(dst, "file1"))); Assert.True(fileSys.Exists(new Path(dst, "file2"))); NUnit.Framework.Assert.IsFalse(fileSys.Exists(src)); }
/// <exception cref="System.Exception"/> public virtual void TestSetTimes() { Path path = new Path(TestRootDir, "set-times"); FileSystemTestHelper.WriteFile(fileSys, path, 1); // test only to the nearest second, as the raw FS may not // support millisecond timestamps long newModTime = 12345000; FileStatus status = fileSys.GetFileStatus(path); Assert.True("check we're actually changing something", newModTime != status.GetModificationTime()); long accessTime = status.GetAccessTime(); fileSys.SetTimes(path, newModTime, -1); status = fileSys.GetFileStatus(path); Assert.Equal(newModTime, status.GetModificationTime()); Assert.Equal(accessTime, status.GetAccessTime()); }
/// <exception cref="System.IO.IOException"/> public virtual void TestCreateFileAndMkdirs() { Path test_dir = new Path(TestRootDir, "test_dir"); Path test_file = new Path(test_dir, "file1"); Assert.True(fileSys.Mkdirs(test_dir)); int fileSize = new Random().Next(1 << 20) + 1; FileSystemTestHelper.WriteFile(fileSys, test_file, fileSize); { //check FileStatus and ContentSummary FileStatus status = fileSys.GetFileStatus(test_file); Assert.Equal(fileSize, status.GetLen()); ContentSummary summary = fileSys.GetContentSummary(test_dir); Assert.Equal(fileSize, summary.GetLength()); } // creating dir over a file Path bad_dir = new Path(test_file, "another_dir"); try { fileSys.Mkdirs(bad_dir); NUnit.Framework.Assert.Fail("Failed to detect existing file in path"); } catch (ParentNotDirectoryException) { } // Expected try { fileSys.Mkdirs(null); NUnit.Framework.Assert.Fail("Failed to detect null in mkdir arg"); } catch (ArgumentException) { } }
/// <summary> /// Test trash for the shell's delete command for the default file system /// specified in the paramter conf /// </summary> /// <param name="conf"></param> /// <param name="base">- the base path where files are created</param> /// <param name="trashRoot">- the expected place where the trashbin resides</param> /// <exception cref="System.IO.IOException"/> public static void TrashShell(Configuration conf, Path @base, FileSystem trashRootFs , Path trashRoot) { FileSystem fs = FileSystem.Get(conf); conf.SetLong(FsTrashIntervalKey, 0); // disabled NUnit.Framework.Assert.IsFalse(new Trash(conf).IsEnabled()); conf.SetLong(FsTrashIntervalKey, 10); // 10 minute Assert.True(new Trash(conf).IsEnabled()); FsShell shell = new FsShell(); shell.SetConf(conf); if (trashRoot == null) { trashRoot = shell.GetCurrentTrashDir(); } if (trashRootFs == null) { trashRootFs = fs; } // First create a new directory with mkdirs Path myPath = new Path(@base, "test/mkdirs"); Mkdir(fs, myPath); // Second, create a file in that directory. Path myFile = new Path(@base, "test/mkdirs/myFile"); FileSystemTestHelper.WriteFile(fs, myFile, 10); { // Verify that expunge without Trash directory // won't throw Exception string[] args = new string[1]; args[0] = "-expunge"; int val = -1; try { val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } Assert.True(val == 0); } { // Verify that we succeed in removing the file we created. // This should go into Trash. string[] args = new string[2]; args[0] = "-rm"; args[1] = myFile.ToString(); int val = -1; try { val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } Assert.True(val == 0); CheckTrash(trashRootFs, trashRoot, fs.MakeQualified(myFile)); } // Verify that we can recreate the file FileSystemTestHelper.WriteFile(fs, myFile, 10); { // Verify that we succeed in removing the file we re-created string[] args = new string[2]; args[0] = "-rm"; args[1] = new Path(@base, "test/mkdirs/myFile").ToString(); int val = -1; try { val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } Assert.True(val == 0); } // Verify that we can recreate the file FileSystemTestHelper.WriteFile(fs, myFile, 10); { // Verify that we succeed in removing the whole directory // along with the file inside it. string[] args = new string[2]; args[0] = "-rmr"; args[1] = new Path(@base, "test/mkdirs").ToString(); int val = -1; try { val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } Assert.True(val == 0); } // recreate directory Mkdir(fs, myPath); { // Verify that we succeed in removing the whole directory string[] args = new string[2]; args[0] = "-rmr"; args[1] = new Path(@base, "test/mkdirs").ToString(); int val = -1; try { val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } Assert.True(val == 0); } { // Check that we can delete a file from the trash Path toErase = new Path(trashRoot, "toErase"); int retVal = -1; FileSystemTestHelper.WriteFile(trashRootFs, toErase, 10); try { retVal = shell.Run(new string[] { "-rm", toErase.ToString() }); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } Assert.True(retVal == 0); CheckNotInTrash(trashRootFs, trashRoot, toErase.ToString()); CheckNotInTrash(trashRootFs, trashRoot, toErase.ToString() + ".1"); } { // simulate Trash removal string[] args = new string[1]; args[0] = "-expunge"; int val = -1; try { val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } Assert.True(val == 0); } // verify that after expunging the Trash, it really goes away CheckNotInTrash(trashRootFs, trashRoot, new Path(@base, "test/mkdirs/myFile").ToString ()); // recreate directory and file Mkdir(fs, myPath); FileSystemTestHelper.WriteFile(fs, myFile, 10); { // remove file first, then remove directory string[] args = new string[2]; args[0] = "-rm"; args[1] = myFile.ToString(); int val = -1; try { val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } Assert.True(val == 0); CheckTrash(trashRootFs, trashRoot, myFile); args = new string[2]; args[0] = "-rmr"; args[1] = myPath.ToString(); val = -1; try { val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } Assert.True(val == 0); CheckTrash(trashRootFs, trashRoot, myPath); } { // attempt to remove parent of trash string[] args = new string[2]; args[0] = "-rmr"; args[1] = trashRoot.GetParent().GetParent().ToString(); int val = -1; try { val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } Assert.Equal("exit code", 1, val); Assert.True(trashRootFs.Exists(trashRoot)); } // Verify skip trash option really works // recreate directory and file Mkdir(fs, myPath); FileSystemTestHelper.WriteFile(fs, myFile, 10); { // Verify that skip trash option really skips the trash for files (rm) string[] args = new string[3]; args[0] = "-rm"; args[1] = "-skipTrash"; args[2] = myFile.ToString(); int val = -1; try { // Clear out trash Assert.Equal("-expunge failed", 0, shell.Run(new string[] { "-expunge" })); val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } NUnit.Framework.Assert.IsFalse("Expected TrashRoot (" + trashRoot + ") to exist in file system:" + trashRootFs.GetUri(), trashRootFs.Exists(trashRoot)); // No new Current should be created NUnit.Framework.Assert.IsFalse(fs.Exists(myFile)); Assert.True(val == 0); } // recreate directory and file Mkdir(fs, myPath); FileSystemTestHelper.WriteFile(fs, myFile, 10); { // Verify that skip trash option really skips the trash for rmr string[] args = new string[3]; args[0] = "-rmr"; args[1] = "-skipTrash"; args[2] = myPath.ToString(); int val = -1; try { // Clear out trash Assert.Equal(0, shell.Run(new string[] { "-expunge" })); val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } NUnit.Framework.Assert.IsFalse(trashRootFs.Exists(trashRoot)); // No new Current should be created NUnit.Framework.Assert.IsFalse(fs.Exists(myPath)); NUnit.Framework.Assert.IsFalse(fs.Exists(myFile)); Assert.True(val == 0); } { // deleting same file multiple times int val = -1; Mkdir(fs, myPath); try { Assert.Equal(0, shell.Run(new string[] { "-expunge" })); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from fs expunge " + e.GetLocalizedMessage ()); } // create a file in that directory. myFile = new Path(@base, "test/mkdirs/myFile"); string[] args = new string[] { "-rm", myFile.ToString() }; int num_runs = 10; for (int i = 0; i < num_runs; i++) { //create file FileSystemTestHelper.WriteFile(fs, myFile, 10); // delete file try { val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } Assert.True(val == 0); } // current trash directory Path trashDir = Path.MergePaths(new Path(trashRoot.ToUri().GetPath()), new Path(myFile .GetParent().ToUri().GetPath())); System.Console.Out.WriteLine("Deleting same myFile: myFile.parent=" + myFile.GetParent ().ToUri().GetPath() + "; trashroot=" + trashRoot.ToUri().GetPath() + "; trashDir=" + trashDir.ToUri().GetPath()); int count = CountSameDeletedFiles(fs, trashDir, myFile); System.Console.Out.WriteLine("counted " + count + " files " + myFile.GetName() + "* in " + trashDir); Assert.True(count == num_runs); } { //Verify skipTrash option is suggested when rm fails due to its absence string[] args = new string[2]; args[0] = "-rmr"; args[1] = "/"; //This always contains trash directory TextWriter stdout = System.Console.Out; TextWriter stderr = System.Console.Error; ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); TextWriter newOut = new TextWriter(byteStream); Runtime.SetOut(newOut); Runtime.SetErr(newOut); try { shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } string output = byteStream.ToString(); Runtime.SetOut(stdout); Runtime.SetErr(stderr); Assert.True("skipTrash wasn't suggested as remedy to failed rm command" + " or we deleted / even though we could not get server defaults", output.IndexOf ("Consider using -skipTrash option") != -1 || output.IndexOf("Failed to determine server trash configuration" ) != -1); } { // Verify old checkpoint format is recognized // emulate two old trash checkpoint directories, one that is old enough // to be deleted on the next expunge and one that isn't. long trashInterval = conf.GetLong(FsTrashIntervalKey, FsTrashIntervalDefault); long now = Time.Now(); DateFormat oldCheckpointFormat = new SimpleDateFormat("yyMMddHHmm"); Path dirToDelete = new Path(trashRoot.GetParent(), oldCheckpointFormat.Format(now - (trashInterval * 60 * 1000) - 1)); Path dirToKeep = new Path(trashRoot.GetParent(), oldCheckpointFormat.Format(now)); Mkdir(trashRootFs, dirToDelete); Mkdir(trashRootFs, dirToKeep); // Clear out trash int rc = -1; try { rc = shell.Run(new string[] { "-expunge" }); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from fs expunge " + e.GetLocalizedMessage ()); } Assert.Equal(0, rc); NUnit.Framework.Assert.IsFalse("old checkpoint format not recognized", trashRootFs .Exists(dirToDelete)); Assert.True("old checkpoint format directory should not be removed" , trashRootFs.Exists(dirToKeep)); } }
/// <summary> /// test same file deletion - multiple time /// this is more of a performance test - shouldn't be run as a unit test /// </summary> /// <exception cref="System.IO.IOException"/> public static void PerformanceTestDeleteSameFile() { Path @base = TestDir; Configuration conf = new Configuration(); conf.SetClass("fs.file.impl", typeof(TestTrash.TestLFS), typeof(FileSystem)); FileSystem fs = FileSystem.GetLocal(conf); conf.Set("fs.defaultFS", fs.GetUri().ToString()); conf.SetLong(FsTrashIntervalKey, 10); //minutes.. FsShell shell = new FsShell(); shell.SetConf(conf); //Path trashRoot = null; Path myPath = new Path(@base, "test/mkdirs"); Mkdir(fs, myPath); // create a file in that directory. Path myFile; long start; long first = 0; int retVal = 0; int factor = 10; // how much slower any of subsequent deletion can be myFile = new Path(@base, "test/mkdirs/myFile"); string[] args = new string[] { "-rm", myFile.ToString() }; int iters = 1000; for (int i = 0; i < iters; i++) { FileSystemTestHelper.WriteFile(fs, myFile, 10); start = Time.Now(); try { retVal = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); throw new IOException(e.Message); } Assert.True(retVal == 0); long iterTime = Time.Now() - start; // take median of the first 10 runs if (i < 10) { if (i == 0) { first = iterTime; } else { first = (first + iterTime) / 2; } } // we don't want to print every iteration - let's do every 10th int print_freq = iters / 10; if (i > 10) { if ((i % print_freq) == 0) { System.Console.Out.WriteLine("iteration=" + i + ";res =" + retVal + "; start=" + start + "; iterTime = " + iterTime + " vs. firstTime=" + first); } long factoredTime = first * factor; Assert.True(iterTime < factoredTime); } } }
/// <exception cref="System.Exception"/> public virtual void TestTrashEmptier() { Configuration conf = new Configuration(); // Trash with 12 second deletes and 6 seconds checkpoints conf.Set(FsTrashIntervalKey, "0.2"); // 12 seconds conf.SetClass("fs.file.impl", typeof(TestTrash.TestLFS), typeof(FileSystem)); conf.Set(FsTrashCheckpointIntervalKey, "0.1"); // 6 seconds FileSystem fs = FileSystem.GetLocal(conf); conf.Set("fs.default.name", fs.GetUri().ToString()); Trash trash = new Trash(conf); // Start Emptier in background Runnable emptier = trash.GetEmptier(); Thread emptierThread = new Thread(emptier); emptierThread.Start(); FsShell shell = new FsShell(); shell.SetConf(conf); shell.Init(); // First create a new directory with mkdirs Path myPath = new Path(TestDir, "test/mkdirs"); Mkdir(fs, myPath); int fileIndex = 0; ICollection <string> checkpoints = new HashSet <string>(); while (true) { // Create a file with a new name Path myFile = new Path(TestDir, "test/mkdirs/myFile" + fileIndex++); FileSystemTestHelper.WriteFile(fs, myFile, 10); // Delete the file to trash string[] args = new string[2]; args[0] = "-rm"; args[1] = myFile.ToString(); int val = -1; try { val = shell.Run(args); } catch (Exception e) { System.Console.Error.WriteLine("Exception raised from Trash.run " + e.GetLocalizedMessage ()); } Assert.True(val == 0); Path trashDir = shell.GetCurrentTrashDir(); FileStatus[] files = fs.ListStatus(trashDir.GetParent()); // Scan files in .Trash and add them to set of checkpoints foreach (FileStatus file in files) { string fileName = file.GetPath().GetName(); checkpoints.AddItem(fileName); } // If checkpoints has 4 objects it is Current + 3 checkpoint directories if (checkpoints.Count == 4) { // The actual contents should be smaller since the last checkpoint // should've been deleted and Current might not have been recreated yet Assert.True(checkpoints.Count > files.Length); break; } Thread.Sleep(5000); } emptierThread.Interrupt(); emptierThread.Join(); }