Пример #1
0
 /// <summary>Skip a test if a feature is unsupported in this FS</summary>
 /// <param name="feature">feature to look for</param>
 /// <exception cref="System.IO.IOException">IO problem</exception>
 protected internal virtual void SkipIfUnsupported(string feature)
 {
     if (!IsSupported(feature))
     {
         ContractTestUtils.Skip("Skipping as unsupported feature: " + feature);
     }
 }
        public virtual void TestOverwriteEmptyDirectory()
        {
            Describe("verify trying to create a file over an empty dir fails");
            Path path = Path("testOverwriteEmptyDirectory");

            Mkdirs(path);
            AssertIsDirectory(path);
            byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
            try
            {
                ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024, true
                                               );
                AssertIsDirectory(path);
                NUnit.Framework.Assert.Fail("write of file over empty dir succeeded");
            }
            catch (FileAlreadyExistsException expected)
            {
                //expected
                HandleExpectedException(expected);
            }
            catch (FileNotFoundException e)
            {
                HandleRelaxedException("overwriting a dir with a file ", "FileAlreadyExistsException"
                                       , e);
            }
            catch (IOException e)
            {
                HandleRelaxedException("overwriting a dir with a file ", "FileAlreadyExistsException"
                                       , e);
            }
            AssertIsDirectory(path);
        }
        public virtual void TestSeekBigFile()
        {
            Describe("Seek round a large file and verify the bytes are what is expected");
            Path testSeekFile = Path("bigseekfile.txt");

            byte[] block = ContractTestUtils.Dataset(65536, 0, 255);
            ContractTestUtils.CreateFile(GetFileSystem(), testSeekFile, false, block);
            instream = GetFileSystem().Open(testSeekFile);
            Assert.Equal(0, instream.GetPos());
            //expect that seek to 0 works
            instream.Seek(0);
            int result = instream.Read();

            Assert.Equal(0, result);
            Assert.Equal(1, instream.Read());
            Assert.Equal(2, instream.Read());
            //do seek 32KB ahead
            instream.Seek(32768);
            Assert.Equal("@32768", block[32768], unchecked ((byte)instream.
                                                            Read()));
            instream.Seek(40000);
            Assert.Equal("@40000", block[40000], unchecked ((byte)instream.
                                                            Read()));
            instream.Seek(8191);
            Assert.Equal("@8191", block[8191], unchecked ((byte)instream.Read
                                                              ()));
            instream.Seek(0);
            Assert.Equal("@0", 0, unchecked ((byte)instream.Read()));
        }
        public virtual void TestCreateFileOverExistingFileNoOverwrite()
        {
            Describe("Verify overwriting an existing file fails");
            Path path = Path("testCreateFileOverExistingFileNoOverwrite");

            byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
            ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024, false
                                           );
            byte[] data2 = ContractTestUtils.Dataset(10 * 1024, 'A', 'Z');
            try
            {
                ContractTestUtils.WriteDataset(GetFileSystem(), path, data2, data2.Length, 1024,
                                               false);
                NUnit.Framework.Assert.Fail("writing without overwrite unexpectedly succeeded");
            }
            catch (FileAlreadyExistsException expected)
            {
                //expected
                HandleExpectedException(expected);
            }
            catch (IOException relaxed)
            {
                HandleRelaxedException("Creating a file over a file with overwrite==false", "FileAlreadyExistsException"
                                       , relaxed);
            }
        }
        public virtual void TestCreatedFileIsImmediatelyVisible()
        {
            Describe("verify that a newly created file exists as soon as open returns");
            Path path = Path("testCreatedFileIsImmediatelyVisible");
            FSDataOutputStream @out = null;

            try
            {
                @out = GetFileSystem().Create(path, false, 4096, (short)1, 1024);
                if (!GetFileSystem().Exists(path))
                {
                    if (IsSupported(IsBlobstore))
                    {
                        // object store: downgrade to a skip so that the failure is visible
                        // in test results
                        ContractTestUtils.Skip("Filesystem is an object store and newly created files are not immediately visible"
                                               );
                    }
                    AssertPathExists("expected path to be visible before anything written", path);
                }
            }
            finally
            {
                IOUtils.CloseStream(@out);
            }
        }
        public virtual void TestOpenFileTwice()
        {
            Describe("verify that two opened file streams are independent");
            Path path = Path("testopenfiletwice.txt");

            byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255);
            //this file now has a simple rule: offset => value
            ContractTestUtils.CreateFile(GetFileSystem(), path, false, block);
            //open first
            FSDataInputStream instream1 = GetFileSystem().Open(path);
            int c = instream1.Read();

            Assert.Equal(0, c);
            FSDataInputStream instream2 = null;

            try
            {
                instream2 = GetFileSystem().Open(path);
                Assert.Equal("first read of instream 2", 0, instream2.Read());
                Assert.Equal("second read of instream 1", 1, instream1.Read());
                instream1.Close();
                Assert.Equal("second read of instream 2", 1, instream2.Read());
                //close instream1 again
                instream1.Close();
            }
            finally
            {
                IOUtils.CloseStream(instream1);
                IOUtils.CloseStream(instream2);
            }
        }
Пример #7
0
        public virtual void TestRenameFileNonexistentDir()
        {
            Describe("rename a file into a new file in the same directory");
            Path renameSrc    = Path("testRenameSrc");
            Path renameTarget = Path("subdir/testRenameTarget");

            byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
            ContractTestUtils.WriteDataset(GetFileSystem(), renameSrc, data, data.Length, 1024
                                           * 1024, false);
            bool renameCreatesDestDirs = IsSupported(RenameCreatesDestDirs);

            try
            {
                bool rename = Rename(renameSrc, renameTarget);
                if (renameCreatesDestDirs)
                {
                    Assert.True(rename);
                    ContractTestUtils.VerifyFileContents(GetFileSystem(), renameTarget, data);
                }
                else
                {
                    NUnit.Framework.Assert.IsFalse(rename);
                    ContractTestUtils.VerifyFileContents(GetFileSystem(), renameSrc, data);
                }
            }
            catch (FileNotFoundException)
            {
                // allowed unless that rename flag is set
                NUnit.Framework.Assert.IsFalse(renameCreatesDestDirs);
            }
        }
Пример #8
0
        public virtual void TestRenameDirIntoExistingDir()
        {
            Describe("Verify renaming a dir into an existing dir puts it underneath" + " and leaves existing files alone"
                     );
            FileSystem fs           = GetFileSystem();
            string     sourceSubdir = "source";
            Path       srcDir       = Path(sourceSubdir);
            Path       srcFilePath  = new Path(srcDir, "source-256.txt");

            byte[] srcDataset = ContractTestUtils.Dataset(256, 'a', 'z');
            ContractTestUtils.WriteDataset(fs, srcFilePath, srcDataset, srcDataset.Length, 1024
                                           , false);
            Path destDir      = Path("dest");
            Path destFilePath = new Path(destDir, "dest-512.txt");

            byte[] destDateset = ContractTestUtils.Dataset(512, 'A', 'Z');
            ContractTestUtils.WriteDataset(fs, destFilePath, destDateset, destDateset.Length,
                                           1024, false);
            AssertIsFile(destFilePath);
            bool rename     = Rename(srcDir, destDir);
            Path renamedSrc = new Path(destDir, sourceSubdir);

            AssertIsFile(destFilePath);
            AssertIsDirectory(renamedSrc);
            ContractTestUtils.VerifyFileContents(fs, destFilePath, destDateset);
            Assert.True("rename returned false though the contents were copied"
                        , rename);
        }
        public virtual void TestRmNonEmptyRootDirNonRecursive()
        {
            //extra sanity checks here to avoid support calls about complete loss of data
            SkipIfUnsupported(TestRootTestsEnabled);
            Path   root      = new Path("/");
            string touchfile = "/testRmNonEmptyRootDirNonRecursive";
            Path   file      = new Path(touchfile);

            ContractTestUtils.Touch(GetFileSystem(), file);
            ContractTestUtils.AssertIsDirectory(GetFileSystem(), root);
            try
            {
                bool deleted = GetFileSystem().Delete(root, false);
                NUnit.Framework.Assert.Fail("non recursive delete should have raised an exception,"
                                            + " but completed with exit code " + deleted);
            }
            catch (IOException e)
            {
                //expected
                HandleExpectedException(e);
            }
            finally
            {
                GetFileSystem().Delete(file, false);
            }
            ContractTestUtils.AssertIsDirectory(GetFileSystem(), root);
        }
 public virtual void TestConcatFileOnFile()
 {
     byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255);
     ContractTestUtils.CreateFile(GetFileSystem(), target, false, block);
     GetFileSystem().Concat(target, new Path[] { srcFile });
     ContractTestUtils.AssertFileHasLength(GetFileSystem(), target, TestFileLen * 2);
     ContractTestUtils.ValidateFileContent(ContractTestUtils.ReadDataset(GetFileSystem
                                                                             (), target, TestFileLen * 2), new byte[][] { block, block });
 }
        public virtual void TestDeleteNonexistentPathNonRecursive()
        {
            Path path = Path("testDeleteNonexistentPathNonRecursive");

            ContractTestUtils.AssertPathDoesNotExist(GetFileSystem(), "leftover", path);
            ContractTestUtils.RejectRootOperation(path);
            NUnit.Framework.Assert.IsFalse("Returned true attempting to recursively delete" +
                                           " a nonexistent path " + path, GetFileSystem().Delete(path, false));
        }
        public virtual void TestOverwriteNonEmptyDirectory()
        {
            Describe("verify trying to create a file over a non-empty dir fails");
            Path path = Path("testOverwriteNonEmptyDirectory");

            Mkdirs(path);
            try
            {
                AssertIsDirectory(path);
            }
            catch (Exception failure)
            {
                if (IsSupported(IsBlobstore))
                {
                    // file/directory hack surfaces here
                    throw Extensions.InitCause(new AssumptionViolatedException(failure.ToString
                                                                                   ()), failure);
                }
                // else: rethrow
                throw;
            }
            Path child = new Path(path, "child");

            ContractTestUtils.WriteTextFile(GetFileSystem(), child, "child file", true);
            byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
            try
            {
                ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024, true
                                               );
                FileStatus status = GetFileSystem().GetFileStatus(path);
                bool       isDir  = status.IsDirectory();
                if (!isDir && IsSupported(IsBlobstore))
                {
                    // object store: downgrade to a skip so that the failure is visible
                    // in test results
                    ContractTestUtils.Skip("Object store allows a file to overwrite a directory");
                }
                NUnit.Framework.Assert.Fail("write of file over dir succeeded");
            }
            catch (FileAlreadyExistsException expected)
            {
                //expected
                HandleExpectedException(expected);
            }
            catch (FileNotFoundException e)
            {
                HandleRelaxedException("overwriting a dir with a file ", "FileAlreadyExistsException"
                                       , e);
            }
            catch (IOException e)
            {
                HandleRelaxedException("overwriting a dir with a file ", "FileAlreadyExistsException"
                                       , e);
            }
            AssertIsDirectory(path);
            AssertIsFile(child);
        }
        public virtual void TestCreateNewFile()
        {
            Describe("Foundational 'create a file' test");
            Path path = Path("testCreateNewFile");

            byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
            ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024 * 1024
                                           , false);
            ContractTestUtils.VerifyFileContents(GetFileSystem(), path, data);
        }
        public virtual void TestMkDirDepth1()
        {
            FileSystem fs  = GetFileSystem();
            Path       dir = new Path("/testmkdirdepth1");

            AssertPathDoesNotExist("directory already exists", dir);
            fs.Mkdirs(dir);
            ContractTestUtils.AssertIsDirectory(GetFileSystem(), dir);
            AssertPathExists("directory already exists", dir);
            AssertDeleted(dir, true);
        }
        public virtual void TestFsIsEncrypted()
        {
            Describe("create an empty file and call FileStatus.isEncrypted()");
            Path path = Path("file");

            ContractTestUtils.CreateFile(GetFileSystem(), path, false, new byte[0]);
            FileStatus stat = GetFileSystem().GetFileStatus(path);

            NUnit.Framework.Assert.IsFalse("Expecting false for stat.isEncrypted()", stat.IsEncrypted
                                               ());
        }
        public virtual void TestDeleteNonEmptyDirRecursive()
        {
            Path path = Path("testDeleteNonEmptyDirNonRecursive");

            Mkdirs(path);
            Path file = new Path(path, "childfile");

            ContractTestUtils.WriteTextFile(GetFileSystem(), file, "goodbye, world", true);
            AssertDeleted(path, true);
            ContractTestUtils.AssertPathDoesNotExist(GetFileSystem(), "not deleted", file);
        }
        public virtual void TestRmEmptyRootDirNonRecursive()
        {
            //extra sanity checks here to avoid support calls about complete loss of data
            SkipIfUnsupported(TestRootTestsEnabled);
            Path root = new Path("/");

            ContractTestUtils.AssertIsDirectory(GetFileSystem(), root);
            bool deleted = GetFileSystem().Delete(root, true);

            Log.Info("rm / of empty dir result is {}", deleted);
            ContractTestUtils.AssertIsDirectory(GetFileSystem(), root);
        }
        public virtual void TestOpenReadZeroByteFile()
        {
            Describe("create & read a 0 byte file");
            Path path = Path("zero.txt");

            ContractTestUtils.Touch(GetFileSystem(), path);
            instream = GetFileSystem().Open(path);
            Assert.Equal(0, instream.GetPos());
            //expect initial read to fail
            int result = instream.Read();

            AssertMinusOne("initial byte read", result);
        }
        public virtual void TestDeleteDeepEmptyDir()
        {
            Mkdirs(Path("testDeleteDeepEmptyDir/d1/d2/d3/d4"));
            AssertDeleted(Path("testDeleteDeepEmptyDir/d1/d2/d3"), true);
            FileSystem fs = GetFileSystem();

            ContractTestUtils.AssertPathDoesNotExist(fs, "not deleted", Path("testDeleteDeepEmptyDir/d1/d2/d3/d4"
                                                                             ));
            ContractTestUtils.AssertPathDoesNotExist(fs, "not deleted", Path("testDeleteDeepEmptyDir/d1/d2/d3"
                                                                             ));
            ContractTestUtils.AssertPathExists(fs, "parent dir is deleted", Path("testDeleteDeepEmptyDir/d1/d2"
                                                                                 ));
        }
 /// <exception cref="System.Exception"/>
 public override void Setup()
 {
     base.Setup();
     SkipIfUnsupported(SupportsSeek);
     //delete the test directory
     testPath      = GetContract().GetTestPath();
     smallSeekFile = Path("seekfile.txt");
     zeroByteFile  = Path("zero.txt");
     byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255);
     //this file now has a simple rule: offset => value
     ContractTestUtils.CreateFile(GetFileSystem(), smallSeekFile, false, block);
     ContractTestUtils.Touch(GetFileSystem(), zeroByteFile);
 }
 /// <exception cref="System.Exception"/>
 public override void Setup()
 {
     base.Setup();
     SkipIfUnsupported(SupportsConcat);
     //delete the test directory
     testPath     = Path("test");
     srcFile      = new Path(testPath, "small.txt");
     zeroByteFile = new Path(testPath, "zero.txt");
     target       = new Path(testPath, "target");
     byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255);
     ContractTestUtils.CreateFile(GetFileSystem(), srcFile, false, block);
     ContractTestUtils.Touch(GetFileSystem(), zeroByteFile);
 }
Пример #22
0
        public virtual void TestAppendToExistingFile()
        {
            byte[] original = ContractTestUtils.Dataset(8192, 'A', 'Z');
            byte[] appended = ContractTestUtils.Dataset(8192, '0', '9');
            ContractTestUtils.CreateFile(GetFileSystem(), target, false, original);
            FSDataOutputStream outputStream = GetFileSystem().Append(target);

            outputStream.Write(appended);
            outputStream.Close();
            byte[] bytes = ContractTestUtils.ReadDataset(GetFileSystem(), target, original.Length
                                                         + appended.Length);
            ContractTestUtils.ValidateFileContent(bytes, new byte[][] { original, appended });
        }
Пример #23
0
        public virtual void TestRenameFileOverExistingFile()
        {
            Describe("Verify renaming a file onto an existing file matches expectations");
            Path srcFile = Path("source-256.txt");

            byte[] srcData = ContractTestUtils.Dataset(256, 'a', 'z');
            ContractTestUtils.WriteDataset(GetFileSystem(), srcFile, srcData, srcData.Length,
                                           1024, false);
            Path destFile = Path("dest-512.txt");

            byte[] destData = ContractTestUtils.Dataset(512, 'A', 'Z');
            ContractTestUtils.WriteDataset(GetFileSystem(), destFile, destData, destData.Length
                                           , 1024, false);
            AssertIsFile(destFile);
            bool renameOverwritesDest = IsSupported(RenameOverwritesDest);
            bool renameReturnsFalseOnRenameDestExists = !IsSupported(RenameReturnsFalseIfDestExists
                                                                     );
            bool destUnchanged = true;

            try
            {
                bool renamed = Rename(srcFile, destFile);
                if (renameOverwritesDest)
                {
                    // the filesystem supports rename(file, file2) by overwriting file2
                    Assert.True("Rename returned false", renamed);
                    destUnchanged = false;
                }
                else
                {
                    // rename is rejected by returning 'false' or throwing an exception
                    if (renamed && !renameReturnsFalseOnRenameDestExists)
                    {
                        //expected an exception
                        string destDirLS = GenerateAndLogErrorListing(srcFile, destFile);
                        GetLog().Error("dest dir {}", destDirLS);
                        NUnit.Framework.Assert.Fail("expected rename(" + srcFile + ", " + destFile + " ) to fail,"
                                                    + " but got success and destination of " + destDirLS);
                    }
                }
            }
            catch (FileAlreadyExistsException e)
            {
                HandleExpectedException(e);
            }
            // verify that the destination file is as expected based on the expected
            // outcome
            ContractTestUtils.VerifyFileContents(GetFileSystem(), destFile, destUnchanged ? destData
                                 : srcData);
        }
        public virtual void TestOverwriteExistingFile()
        {
            Describe("Overwrite an existing file and verify the new data is there");
            Path path = Path("testOverwriteExistingFile");

            byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
            ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024, false
                                           );
            ContractTestUtils.VerifyFileContents(GetFileSystem(), path, data);
            byte[] data2 = ContractTestUtils.Dataset(10 * 1024, 'A', 'Z');
            ContractTestUtils.WriteDataset(GetFileSystem(), path, data2, data2.Length, 1024,
                                           true);
            ContractTestUtils.VerifyFileContents(GetFileSystem(), path, data2);
        }
        public virtual void TestDeleteSingleFile()
        {
            // Test delete of just a file
            Path path = Path("testDeleteSingleFile/d1/d2");

            Mkdirs(path);
            Path file = new Path(path, "childfile");

            ContractTestUtils.WriteTextFile(GetFileSystem(), file, "single file to be deleted."
                                            , true);
            ContractTestUtils.AssertPathExists(GetFileSystem(), "single file not created", file
                                               );
            AssertDeleted(file, false);
        }
 public virtual void TestConcatEmptyFiles()
 {
     ContractTestUtils.Touch(GetFileSystem(), target);
     try
     {
         GetFileSystem().Concat(target, new Path[0]);
         NUnit.Framework.Assert.Fail("expected a failure");
     }
     catch (Exception e)
     {
         //expected
         HandleExpectedException(e);
     }
 }
 public virtual void TestConcatOnSelf()
 {
     byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255);
     ContractTestUtils.CreateFile(GetFileSystem(), target, false, block);
     try
     {
         GetFileSystem().Concat(target, new Path[] { target });
     }
     catch (Exception e)
     {
         //expected
         HandleExpectedException(e);
     }
 }
Пример #28
0
        /// <exception cref="System.IO.IOException"/>
        protected internal virtual string GenerateAndLogErrorListing(Org.Apache.Hadoop.FS.Path
                                                                     src, Org.Apache.Hadoop.FS.Path dst)
        {
            FileSystem fs = GetFileSystem();

            GetLog().Error("src dir " + ContractTestUtils.Ls(fs, src.GetParent()));
            string destDirLS = ContractTestUtils.Ls(fs, dst.GetParent());

            if (fs.IsDirectory(dst))
            {
                //include the dir into the listing
                destDirLS = destDirLS + "\n" + ContractTestUtils.Ls(fs, dst);
            }
            return(destDirLS);
        }
Пример #29
0
        public virtual void TestRenameNewFileSameDir()
        {
            Describe("rename a file into a new file in the same directory");
            Path renameSrc    = Path("rename_src");
            Path renameTarget = Path("rename_dest");

            byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
            ContractTestUtils.WriteDataset(GetFileSystem(), renameSrc, data, data.Length, 1024
                                           * 1024, false);
            bool rename = Rename(renameSrc, renameTarget);

            Assert.True("rename(" + renameSrc + ", " + renameTarget + ") returned false"
                        , rename);
            ContractTestUtils.AssertListStatusFinds(GetFileSystem(), renameTarget.GetParent()
                                                    , renameTarget);
            ContractTestUtils.VerifyFileContents(GetFileSystem(), renameTarget, data);
        }
        public virtual void TestRandomSeeks()
        {
            int limit = GetContract().GetLimit(TestRandomSeekCount, DefaultRandomSeekCount);

            Describe("Testing " + limit + " random seeks");
            int filesize = 10 * 1024;

            byte[] buf            = ContractTestUtils.Dataset(filesize, 0, 255);
            Path   randomSeekFile = Path("testrandomseeks.bin");

            ContractTestUtils.CreateFile(GetFileSystem(), randomSeekFile, false, buf);
            Random            r   = new Random();
            FSDataInputStream stm = GetFileSystem().Open(randomSeekFile);

            // Record the sequence of seeks and reads which trigger a failure.
            int[] seeks = new int[10];
            int[] reads = new int[10];
            try
            {
                for (int i = 0; i < limit; i++)
                {
                    int seekOff = r.Next(buf.Length);
                    int toRead  = r.Next(Math.Min(buf.Length - seekOff, 32000));
                    seeks[i % seeks.Length] = seekOff;
                    reads[i % reads.Length] = toRead;
                    ContractTestUtils.VerifyRead(stm, buf, seekOff, toRead);
                }
            }
            catch (Exception afe)
            {
                StringBuilder sb = new StringBuilder();
                sb.Append("Sequence of actions:\n");
                for (int j = 0; j < seeks.Length; j++)
                {
                    sb.Append("seek @ ").Append(seeks[j]).Append("  ").Append("read ").Append(reads[j
                                                                                              ]).Append("\n");
                }
                Log.Error(sb.ToString());
                throw;
            }
            finally
            {
                stm.Close();
            }
        }