/// <summary>Skip a test if a feature is unsupported in this FS</summary> /// <param name="feature">feature to look for</param> /// <exception cref="System.IO.IOException">IO problem</exception> protected internal virtual void SkipIfUnsupported(string feature) { if (!IsSupported(feature)) { ContractTestUtils.Skip("Skipping as unsupported feature: " + feature); } }
public virtual void TestOverwriteEmptyDirectory() { Describe("verify trying to create a file over an empty dir fails"); Path path = Path("testOverwriteEmptyDirectory"); Mkdirs(path); AssertIsDirectory(path); byte[] data = ContractTestUtils.Dataset(256, 'a', 'z'); try { ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024, true ); AssertIsDirectory(path); NUnit.Framework.Assert.Fail("write of file over empty dir succeeded"); } catch (FileAlreadyExistsException expected) { //expected HandleExpectedException(expected); } catch (FileNotFoundException e) { HandleRelaxedException("overwriting a dir with a file ", "FileAlreadyExistsException" , e); } catch (IOException e) { HandleRelaxedException("overwriting a dir with a file ", "FileAlreadyExistsException" , e); } AssertIsDirectory(path); }
public virtual void TestSeekBigFile() { Describe("Seek round a large file and verify the bytes are what is expected"); Path testSeekFile = Path("bigseekfile.txt"); byte[] block = ContractTestUtils.Dataset(65536, 0, 255); ContractTestUtils.CreateFile(GetFileSystem(), testSeekFile, false, block); instream = GetFileSystem().Open(testSeekFile); Assert.Equal(0, instream.GetPos()); //expect that seek to 0 works instream.Seek(0); int result = instream.Read(); Assert.Equal(0, result); Assert.Equal(1, instream.Read()); Assert.Equal(2, instream.Read()); //do seek 32KB ahead instream.Seek(32768); Assert.Equal("@32768", block[32768], unchecked ((byte)instream. Read())); instream.Seek(40000); Assert.Equal("@40000", block[40000], unchecked ((byte)instream. Read())); instream.Seek(8191); Assert.Equal("@8191", block[8191], unchecked ((byte)instream.Read ())); instream.Seek(0); Assert.Equal("@0", 0, unchecked ((byte)instream.Read())); }
public virtual void TestCreateFileOverExistingFileNoOverwrite() { Describe("Verify overwriting an existing file fails"); Path path = Path("testCreateFileOverExistingFileNoOverwrite"); byte[] data = ContractTestUtils.Dataset(256, 'a', 'z'); ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024, false ); byte[] data2 = ContractTestUtils.Dataset(10 * 1024, 'A', 'Z'); try { ContractTestUtils.WriteDataset(GetFileSystem(), path, data2, data2.Length, 1024, false); NUnit.Framework.Assert.Fail("writing without overwrite unexpectedly succeeded"); } catch (FileAlreadyExistsException expected) { //expected HandleExpectedException(expected); } catch (IOException relaxed) { HandleRelaxedException("Creating a file over a file with overwrite==false", "FileAlreadyExistsException" , relaxed); } }
public virtual void TestCreatedFileIsImmediatelyVisible() { Describe("verify that a newly created file exists as soon as open returns"); Path path = Path("testCreatedFileIsImmediatelyVisible"); FSDataOutputStream @out = null; try { @out = GetFileSystem().Create(path, false, 4096, (short)1, 1024); if (!GetFileSystem().Exists(path)) { if (IsSupported(IsBlobstore)) { // object store: downgrade to a skip so that the failure is visible // in test results ContractTestUtils.Skip("Filesystem is an object store and newly created files are not immediately visible" ); } AssertPathExists("expected path to be visible before anything written", path); } } finally { IOUtils.CloseStream(@out); } }
public virtual void TestOpenFileTwice() { Describe("verify that two opened file streams are independent"); Path path = Path("testopenfiletwice.txt"); byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255); //this file now has a simple rule: offset => value ContractTestUtils.CreateFile(GetFileSystem(), path, false, block); //open first FSDataInputStream instream1 = GetFileSystem().Open(path); int c = instream1.Read(); Assert.Equal(0, c); FSDataInputStream instream2 = null; try { instream2 = GetFileSystem().Open(path); Assert.Equal("first read of instream 2", 0, instream2.Read()); Assert.Equal("second read of instream 1", 1, instream1.Read()); instream1.Close(); Assert.Equal("second read of instream 2", 1, instream2.Read()); //close instream1 again instream1.Close(); } finally { IOUtils.CloseStream(instream1); IOUtils.CloseStream(instream2); } }
public virtual void TestRenameFileNonexistentDir() { Describe("rename a file into a new file in the same directory"); Path renameSrc = Path("testRenameSrc"); Path renameTarget = Path("subdir/testRenameTarget"); byte[] data = ContractTestUtils.Dataset(256, 'a', 'z'); ContractTestUtils.WriteDataset(GetFileSystem(), renameSrc, data, data.Length, 1024 * 1024, false); bool renameCreatesDestDirs = IsSupported(RenameCreatesDestDirs); try { bool rename = Rename(renameSrc, renameTarget); if (renameCreatesDestDirs) { Assert.True(rename); ContractTestUtils.VerifyFileContents(GetFileSystem(), renameTarget, data); } else { NUnit.Framework.Assert.IsFalse(rename); ContractTestUtils.VerifyFileContents(GetFileSystem(), renameSrc, data); } } catch (FileNotFoundException) { // allowed unless that rename flag is set NUnit.Framework.Assert.IsFalse(renameCreatesDestDirs); } }
public virtual void TestRenameDirIntoExistingDir() { Describe("Verify renaming a dir into an existing dir puts it underneath" + " and leaves existing files alone" ); FileSystem fs = GetFileSystem(); string sourceSubdir = "source"; Path srcDir = Path(sourceSubdir); Path srcFilePath = new Path(srcDir, "source-256.txt"); byte[] srcDataset = ContractTestUtils.Dataset(256, 'a', 'z'); ContractTestUtils.WriteDataset(fs, srcFilePath, srcDataset, srcDataset.Length, 1024 , false); Path destDir = Path("dest"); Path destFilePath = new Path(destDir, "dest-512.txt"); byte[] destDateset = ContractTestUtils.Dataset(512, 'A', 'Z'); ContractTestUtils.WriteDataset(fs, destFilePath, destDateset, destDateset.Length, 1024, false); AssertIsFile(destFilePath); bool rename = Rename(srcDir, destDir); Path renamedSrc = new Path(destDir, sourceSubdir); AssertIsFile(destFilePath); AssertIsDirectory(renamedSrc); ContractTestUtils.VerifyFileContents(fs, destFilePath, destDateset); Assert.True("rename returned false though the contents were copied" , rename); }
public virtual void TestRmNonEmptyRootDirNonRecursive() { //extra sanity checks here to avoid support calls about complete loss of data SkipIfUnsupported(TestRootTestsEnabled); Path root = new Path("/"); string touchfile = "/testRmNonEmptyRootDirNonRecursive"; Path file = new Path(touchfile); ContractTestUtils.Touch(GetFileSystem(), file); ContractTestUtils.AssertIsDirectory(GetFileSystem(), root); try { bool deleted = GetFileSystem().Delete(root, false); NUnit.Framework.Assert.Fail("non recursive delete should have raised an exception," + " but completed with exit code " + deleted); } catch (IOException e) { //expected HandleExpectedException(e); } finally { GetFileSystem().Delete(file, false); } ContractTestUtils.AssertIsDirectory(GetFileSystem(), root); }
public virtual void TestConcatFileOnFile() { byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255); ContractTestUtils.CreateFile(GetFileSystem(), target, false, block); GetFileSystem().Concat(target, new Path[] { srcFile }); ContractTestUtils.AssertFileHasLength(GetFileSystem(), target, TestFileLen * 2); ContractTestUtils.ValidateFileContent(ContractTestUtils.ReadDataset(GetFileSystem (), target, TestFileLen * 2), new byte[][] { block, block }); }
public virtual void TestDeleteNonexistentPathNonRecursive() { Path path = Path("testDeleteNonexistentPathNonRecursive"); ContractTestUtils.AssertPathDoesNotExist(GetFileSystem(), "leftover", path); ContractTestUtils.RejectRootOperation(path); NUnit.Framework.Assert.IsFalse("Returned true attempting to recursively delete" + " a nonexistent path " + path, GetFileSystem().Delete(path, false)); }
public virtual void TestOverwriteNonEmptyDirectory() { Describe("verify trying to create a file over a non-empty dir fails"); Path path = Path("testOverwriteNonEmptyDirectory"); Mkdirs(path); try { AssertIsDirectory(path); } catch (Exception failure) { if (IsSupported(IsBlobstore)) { // file/directory hack surfaces here throw Extensions.InitCause(new AssumptionViolatedException(failure.ToString ()), failure); } // else: rethrow throw; } Path child = new Path(path, "child"); ContractTestUtils.WriteTextFile(GetFileSystem(), child, "child file", true); byte[] data = ContractTestUtils.Dataset(256, 'a', 'z'); try { ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024, true ); FileStatus status = GetFileSystem().GetFileStatus(path); bool isDir = status.IsDirectory(); if (!isDir && IsSupported(IsBlobstore)) { // object store: downgrade to a skip so that the failure is visible // in test results ContractTestUtils.Skip("Object store allows a file to overwrite a directory"); } NUnit.Framework.Assert.Fail("write of file over dir succeeded"); } catch (FileAlreadyExistsException expected) { //expected HandleExpectedException(expected); } catch (FileNotFoundException e) { HandleRelaxedException("overwriting a dir with a file ", "FileAlreadyExistsException" , e); } catch (IOException e) { HandleRelaxedException("overwriting a dir with a file ", "FileAlreadyExistsException" , e); } AssertIsDirectory(path); AssertIsFile(child); }
public virtual void TestCreateNewFile() { Describe("Foundational 'create a file' test"); Path path = Path("testCreateNewFile"); byte[] data = ContractTestUtils.Dataset(256, 'a', 'z'); ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024 * 1024 , false); ContractTestUtils.VerifyFileContents(GetFileSystem(), path, data); }
public virtual void TestMkDirDepth1() { FileSystem fs = GetFileSystem(); Path dir = new Path("/testmkdirdepth1"); AssertPathDoesNotExist("directory already exists", dir); fs.Mkdirs(dir); ContractTestUtils.AssertIsDirectory(GetFileSystem(), dir); AssertPathExists("directory already exists", dir); AssertDeleted(dir, true); }
public virtual void TestFsIsEncrypted() { Describe("create an empty file and call FileStatus.isEncrypted()"); Path path = Path("file"); ContractTestUtils.CreateFile(GetFileSystem(), path, false, new byte[0]); FileStatus stat = GetFileSystem().GetFileStatus(path); NUnit.Framework.Assert.IsFalse("Expecting false for stat.isEncrypted()", stat.IsEncrypted ()); }
public virtual void TestDeleteNonEmptyDirRecursive() { Path path = Path("testDeleteNonEmptyDirNonRecursive"); Mkdirs(path); Path file = new Path(path, "childfile"); ContractTestUtils.WriteTextFile(GetFileSystem(), file, "goodbye, world", true); AssertDeleted(path, true); ContractTestUtils.AssertPathDoesNotExist(GetFileSystem(), "not deleted", file); }
public virtual void TestRmEmptyRootDirNonRecursive() { //extra sanity checks here to avoid support calls about complete loss of data SkipIfUnsupported(TestRootTestsEnabled); Path root = new Path("/"); ContractTestUtils.AssertIsDirectory(GetFileSystem(), root); bool deleted = GetFileSystem().Delete(root, true); Log.Info("rm / of empty dir result is {}", deleted); ContractTestUtils.AssertIsDirectory(GetFileSystem(), root); }
public virtual void TestOpenReadZeroByteFile() { Describe("create & read a 0 byte file"); Path path = Path("zero.txt"); ContractTestUtils.Touch(GetFileSystem(), path); instream = GetFileSystem().Open(path); Assert.Equal(0, instream.GetPos()); //expect initial read to fail int result = instream.Read(); AssertMinusOne("initial byte read", result); }
public virtual void TestDeleteDeepEmptyDir() { Mkdirs(Path("testDeleteDeepEmptyDir/d1/d2/d3/d4")); AssertDeleted(Path("testDeleteDeepEmptyDir/d1/d2/d3"), true); FileSystem fs = GetFileSystem(); ContractTestUtils.AssertPathDoesNotExist(fs, "not deleted", Path("testDeleteDeepEmptyDir/d1/d2/d3/d4" )); ContractTestUtils.AssertPathDoesNotExist(fs, "not deleted", Path("testDeleteDeepEmptyDir/d1/d2/d3" )); ContractTestUtils.AssertPathExists(fs, "parent dir is deleted", Path("testDeleteDeepEmptyDir/d1/d2" )); }
/// <exception cref="System.Exception"/> public override void Setup() { base.Setup(); SkipIfUnsupported(SupportsSeek); //delete the test directory testPath = GetContract().GetTestPath(); smallSeekFile = Path("seekfile.txt"); zeroByteFile = Path("zero.txt"); byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255); //this file now has a simple rule: offset => value ContractTestUtils.CreateFile(GetFileSystem(), smallSeekFile, false, block); ContractTestUtils.Touch(GetFileSystem(), zeroByteFile); }
/// <exception cref="System.Exception"/> public override void Setup() { base.Setup(); SkipIfUnsupported(SupportsConcat); //delete the test directory testPath = Path("test"); srcFile = new Path(testPath, "small.txt"); zeroByteFile = new Path(testPath, "zero.txt"); target = new Path(testPath, "target"); byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255); ContractTestUtils.CreateFile(GetFileSystem(), srcFile, false, block); ContractTestUtils.Touch(GetFileSystem(), zeroByteFile); }
public virtual void TestAppendToExistingFile() { byte[] original = ContractTestUtils.Dataset(8192, 'A', 'Z'); byte[] appended = ContractTestUtils.Dataset(8192, '0', '9'); ContractTestUtils.CreateFile(GetFileSystem(), target, false, original); FSDataOutputStream outputStream = GetFileSystem().Append(target); outputStream.Write(appended); outputStream.Close(); byte[] bytes = ContractTestUtils.ReadDataset(GetFileSystem(), target, original.Length + appended.Length); ContractTestUtils.ValidateFileContent(bytes, new byte[][] { original, appended }); }
public virtual void TestRenameFileOverExistingFile() { Describe("Verify renaming a file onto an existing file matches expectations"); Path srcFile = Path("source-256.txt"); byte[] srcData = ContractTestUtils.Dataset(256, 'a', 'z'); ContractTestUtils.WriteDataset(GetFileSystem(), srcFile, srcData, srcData.Length, 1024, false); Path destFile = Path("dest-512.txt"); byte[] destData = ContractTestUtils.Dataset(512, 'A', 'Z'); ContractTestUtils.WriteDataset(GetFileSystem(), destFile, destData, destData.Length , 1024, false); AssertIsFile(destFile); bool renameOverwritesDest = IsSupported(RenameOverwritesDest); bool renameReturnsFalseOnRenameDestExists = !IsSupported(RenameReturnsFalseIfDestExists ); bool destUnchanged = true; try { bool renamed = Rename(srcFile, destFile); if (renameOverwritesDest) { // the filesystem supports rename(file, file2) by overwriting file2 Assert.True("Rename returned false", renamed); destUnchanged = false; } else { // rename is rejected by returning 'false' or throwing an exception if (renamed && !renameReturnsFalseOnRenameDestExists) { //expected an exception string destDirLS = GenerateAndLogErrorListing(srcFile, destFile); GetLog().Error("dest dir {}", destDirLS); NUnit.Framework.Assert.Fail("expected rename(" + srcFile + ", " + destFile + " ) to fail," + " but got success and destination of " + destDirLS); } } } catch (FileAlreadyExistsException e) { HandleExpectedException(e); } // verify that the destination file is as expected based on the expected // outcome ContractTestUtils.VerifyFileContents(GetFileSystem(), destFile, destUnchanged ? destData : srcData); }
public virtual void TestOverwriteExistingFile() { Describe("Overwrite an existing file and verify the new data is there"); Path path = Path("testOverwriteExistingFile"); byte[] data = ContractTestUtils.Dataset(256, 'a', 'z'); ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024, false ); ContractTestUtils.VerifyFileContents(GetFileSystem(), path, data); byte[] data2 = ContractTestUtils.Dataset(10 * 1024, 'A', 'Z'); ContractTestUtils.WriteDataset(GetFileSystem(), path, data2, data2.Length, 1024, true); ContractTestUtils.VerifyFileContents(GetFileSystem(), path, data2); }
public virtual void TestDeleteSingleFile() { // Test delete of just a file Path path = Path("testDeleteSingleFile/d1/d2"); Mkdirs(path); Path file = new Path(path, "childfile"); ContractTestUtils.WriteTextFile(GetFileSystem(), file, "single file to be deleted." , true); ContractTestUtils.AssertPathExists(GetFileSystem(), "single file not created", file ); AssertDeleted(file, false); }
public virtual void TestConcatEmptyFiles() { ContractTestUtils.Touch(GetFileSystem(), target); try { GetFileSystem().Concat(target, new Path[0]); NUnit.Framework.Assert.Fail("expected a failure"); } catch (Exception e) { //expected HandleExpectedException(e); } }
public virtual void TestConcatOnSelf() { byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255); ContractTestUtils.CreateFile(GetFileSystem(), target, false, block); try { GetFileSystem().Concat(target, new Path[] { target }); } catch (Exception e) { //expected HandleExpectedException(e); } }
/// <exception cref="System.IO.IOException"/> protected internal virtual string GenerateAndLogErrorListing(Org.Apache.Hadoop.FS.Path src, Org.Apache.Hadoop.FS.Path dst) { FileSystem fs = GetFileSystem(); GetLog().Error("src dir " + ContractTestUtils.Ls(fs, src.GetParent())); string destDirLS = ContractTestUtils.Ls(fs, dst.GetParent()); if (fs.IsDirectory(dst)) { //include the dir into the listing destDirLS = destDirLS + "\n" + ContractTestUtils.Ls(fs, dst); } return(destDirLS); }
public virtual void TestRenameNewFileSameDir() { Describe("rename a file into a new file in the same directory"); Path renameSrc = Path("rename_src"); Path renameTarget = Path("rename_dest"); byte[] data = ContractTestUtils.Dataset(256, 'a', 'z'); ContractTestUtils.WriteDataset(GetFileSystem(), renameSrc, data, data.Length, 1024 * 1024, false); bool rename = Rename(renameSrc, renameTarget); Assert.True("rename(" + renameSrc + ", " + renameTarget + ") returned false" , rename); ContractTestUtils.AssertListStatusFinds(GetFileSystem(), renameTarget.GetParent() , renameTarget); ContractTestUtils.VerifyFileContents(GetFileSystem(), renameTarget, data); }
public virtual void TestRandomSeeks() { int limit = GetContract().GetLimit(TestRandomSeekCount, DefaultRandomSeekCount); Describe("Testing " + limit + " random seeks"); int filesize = 10 * 1024; byte[] buf = ContractTestUtils.Dataset(filesize, 0, 255); Path randomSeekFile = Path("testrandomseeks.bin"); ContractTestUtils.CreateFile(GetFileSystem(), randomSeekFile, false, buf); Random r = new Random(); FSDataInputStream stm = GetFileSystem().Open(randomSeekFile); // Record the sequence of seeks and reads which trigger a failure. int[] seeks = new int[10]; int[] reads = new int[10]; try { for (int i = 0; i < limit; i++) { int seekOff = r.Next(buf.Length); int toRead = r.Next(Math.Min(buf.Length - seekOff, 32000)); seeks[i % seeks.Length] = seekOff; reads[i % reads.Length] = toRead; ContractTestUtils.VerifyRead(stm, buf, seekOff, toRead); } } catch (Exception afe) { StringBuilder sb = new StringBuilder(); sb.Append("Sequence of actions:\n"); for (int j = 0; j < seeks.Length; j++) { sb.Append("seek @ ").Append(seeks[j]).Append(" ").Append("read ").Append(reads[j ]).Append("\n"); } Log.Error(sb.ToString()); throw; } finally { stm.Close(); } }