/// <summary>Skip a test if a feature is unsupported in this FS</summary>
/// <param name="feature">feature to look for</param>
/// <exception cref="System.IO.IOException">IO problem</exception>
protected internal virtual void SkipIfUnsupported(string feature)
{
if (!IsSupported(feature))
{
ContractTestUtils.Skip("Skipping as unsupported feature: " + feature);
}
}
public virtual void TestOverwriteEmptyDirectory()
{
Describe("verify trying to create a file over an empty dir fails");
Path path = Path("testOverwriteEmptyDirectory");
Mkdirs(path);
AssertIsDirectory(path);
byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
try
{
ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024, true
);
AssertIsDirectory(path);
NUnit.Framework.Assert.Fail("write of file over empty dir succeeded");
}
catch (FileAlreadyExistsException expected)
{
//expected
HandleExpectedException(expected);
}
catch (FileNotFoundException e)
{
HandleRelaxedException("overwriting a dir with a file ", "FileAlreadyExistsException"
, e);
}
catch (IOException e)
{
HandleRelaxedException("overwriting a dir with a file ", "FileAlreadyExistsException"
, e);
}
AssertIsDirectory(path);
}
public virtual void TestSeekBigFile()
{
Describe("Seek round a large file and verify the bytes are what is expected");
Path testSeekFile = Path("bigseekfile.txt");
byte[] block = ContractTestUtils.Dataset(65536, 0, 255);
ContractTestUtils.CreateFile(GetFileSystem(), testSeekFile, false, block);
instream = GetFileSystem().Open(testSeekFile);
Assert.Equal(0, instream.GetPos());
//expect that seek to 0 works
instream.Seek(0);
int result = instream.Read();
Assert.Equal(0, result);
Assert.Equal(1, instream.Read());
Assert.Equal(2, instream.Read());
//do seek 32KB ahead
instream.Seek(32768);
Assert.Equal("@32768", block[32768], unchecked ((byte)instream.
Read()));
instream.Seek(40000);
Assert.Equal("@40000", block[40000], unchecked ((byte)instream.
Read()));
instream.Seek(8191);
Assert.Equal("@8191", block[8191], unchecked ((byte)instream.Read
()));
instream.Seek(0);
Assert.Equal("@0", 0, unchecked ((byte)instream.Read()));
}
public virtual void TestCreateFileOverExistingFileNoOverwrite()
{
Describe("Verify overwriting an existing file fails");
Path path = Path("testCreateFileOverExistingFileNoOverwrite");
byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024, false
);
byte[] data2 = ContractTestUtils.Dataset(10 * 1024, 'A', 'Z');
try
{
ContractTestUtils.WriteDataset(GetFileSystem(), path, data2, data2.Length, 1024,
false);
NUnit.Framework.Assert.Fail("writing without overwrite unexpectedly succeeded");
}
catch (FileAlreadyExistsException expected)
{
//expected
HandleExpectedException(expected);
}
catch (IOException relaxed)
{
HandleRelaxedException("Creating a file over a file with overwrite==false", "FileAlreadyExistsException"
, relaxed);
}
}
public virtual void TestCreatedFileIsImmediatelyVisible()
{
Describe("verify that a newly created file exists as soon as open returns");
Path path = Path("testCreatedFileIsImmediatelyVisible");
FSDataOutputStream @out = null;
try
{
@out = GetFileSystem().Create(path, false, 4096, (short)1, 1024);
if (!GetFileSystem().Exists(path))
{
if (IsSupported(IsBlobstore))
{
// object store: downgrade to a skip so that the failure is visible
// in test results
ContractTestUtils.Skip("Filesystem is an object store and newly created files are not immediately visible"
);
}
AssertPathExists("expected path to be visible before anything written", path);
}
}
finally
{
IOUtils.CloseStream(@out);
}
}
public virtual void TestOpenFileTwice()
{
Describe("verify that two opened file streams are independent");
Path path = Path("testopenfiletwice.txt");
byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255);
//this file now has a simple rule: offset => value
ContractTestUtils.CreateFile(GetFileSystem(), path, false, block);
//open first
FSDataInputStream instream1 = GetFileSystem().Open(path);
int c = instream1.Read();
Assert.Equal(0, c);
FSDataInputStream instream2 = null;
try
{
instream2 = GetFileSystem().Open(path);
Assert.Equal("first read of instream 2", 0, instream2.Read());
Assert.Equal("second read of instream 1", 1, instream1.Read());
instream1.Close();
Assert.Equal("second read of instream 2", 1, instream2.Read());
//close instream1 again
instream1.Close();
}
finally
{
IOUtils.CloseStream(instream1);
IOUtils.CloseStream(instream2);
}
}
public virtual void TestRenameFileNonexistentDir()
{
Describe("rename a file into a new file in the same directory");
Path renameSrc = Path("testRenameSrc");
Path renameTarget = Path("subdir/testRenameTarget");
byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
ContractTestUtils.WriteDataset(GetFileSystem(), renameSrc, data, data.Length, 1024
* 1024, false);
bool renameCreatesDestDirs = IsSupported(RenameCreatesDestDirs);
try
{
bool rename = Rename(renameSrc, renameTarget);
if (renameCreatesDestDirs)
{
Assert.True(rename);
ContractTestUtils.VerifyFileContents(GetFileSystem(), renameTarget, data);
}
else
{
NUnit.Framework.Assert.IsFalse(rename);
ContractTestUtils.VerifyFileContents(GetFileSystem(), renameSrc, data);
}
}
catch (FileNotFoundException)
{
// allowed unless that rename flag is set
NUnit.Framework.Assert.IsFalse(renameCreatesDestDirs);
}
}
public virtual void TestRenameDirIntoExistingDir()
{
Describe("Verify renaming a dir into an existing dir puts it underneath" + " and leaves existing files alone"
);
FileSystem fs = GetFileSystem();
string sourceSubdir = "source";
Path srcDir = Path(sourceSubdir);
Path srcFilePath = new Path(srcDir, "source-256.txt");
byte[] srcDataset = ContractTestUtils.Dataset(256, 'a', 'z');
ContractTestUtils.WriteDataset(fs, srcFilePath, srcDataset, srcDataset.Length, 1024
, false);
Path destDir = Path("dest");
Path destFilePath = new Path(destDir, "dest-512.txt");
byte[] destDateset = ContractTestUtils.Dataset(512, 'A', 'Z');
ContractTestUtils.WriteDataset(fs, destFilePath, destDateset, destDateset.Length,
1024, false);
AssertIsFile(destFilePath);
bool rename = Rename(srcDir, destDir);
Path renamedSrc = new Path(destDir, sourceSubdir);
AssertIsFile(destFilePath);
AssertIsDirectory(renamedSrc);
ContractTestUtils.VerifyFileContents(fs, destFilePath, destDateset);
Assert.True("rename returned false though the contents were copied"
, rename);
}
public virtual void TestRmNonEmptyRootDirNonRecursive()
{
//extra sanity checks here to avoid support calls about complete loss of data
SkipIfUnsupported(TestRootTestsEnabled);
Path root = new Path("/");
string touchfile = "/testRmNonEmptyRootDirNonRecursive";
Path file = new Path(touchfile);
ContractTestUtils.Touch(GetFileSystem(), file);
ContractTestUtils.AssertIsDirectory(GetFileSystem(), root);
try
{
bool deleted = GetFileSystem().Delete(root, false);
NUnit.Framework.Assert.Fail("non recursive delete should have raised an exception,"
+ " but completed with exit code " + deleted);
}
catch (IOException e)
{
//expected
HandleExpectedException(e);
}
finally
{
GetFileSystem().Delete(file, false);
}
ContractTestUtils.AssertIsDirectory(GetFileSystem(), root);
}
public virtual void TestConcatFileOnFile()
{
byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255);
ContractTestUtils.CreateFile(GetFileSystem(), target, false, block);
GetFileSystem().Concat(target, new Path[] { srcFile });
ContractTestUtils.AssertFileHasLength(GetFileSystem(), target, TestFileLen * 2);
ContractTestUtils.ValidateFileContent(ContractTestUtils.ReadDataset(GetFileSystem
(), target, TestFileLen * 2), new byte[][] { block, block });
}
public virtual void TestDeleteNonexistentPathNonRecursive()
{
Path path = Path("testDeleteNonexistentPathNonRecursive");
ContractTestUtils.AssertPathDoesNotExist(GetFileSystem(), "leftover", path);
ContractTestUtils.RejectRootOperation(path);
NUnit.Framework.Assert.IsFalse("Returned true attempting to recursively delete" +
" a nonexistent path " + path, GetFileSystem().Delete(path, false));
}
public virtual void TestOverwriteNonEmptyDirectory()
{
Describe("verify trying to create a file over a non-empty dir fails");
Path path = Path("testOverwriteNonEmptyDirectory");
Mkdirs(path);
try
{
AssertIsDirectory(path);
}
catch (Exception failure)
{
if (IsSupported(IsBlobstore))
{
// file/directory hack surfaces here
throw Extensions.InitCause(new AssumptionViolatedException(failure.ToString
()), failure);
}
// else: rethrow
throw;
}
Path child = new Path(path, "child");
ContractTestUtils.WriteTextFile(GetFileSystem(), child, "child file", true);
byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
try
{
ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024, true
);
FileStatus status = GetFileSystem().GetFileStatus(path);
bool isDir = status.IsDirectory();
if (!isDir && IsSupported(IsBlobstore))
{
// object store: downgrade to a skip so that the failure is visible
// in test results
ContractTestUtils.Skip("Object store allows a file to overwrite a directory");
}
NUnit.Framework.Assert.Fail("write of file over dir succeeded");
}
catch (FileAlreadyExistsException expected)
{
//expected
HandleExpectedException(expected);
}
catch (FileNotFoundException e)
{
HandleRelaxedException("overwriting a dir with a file ", "FileAlreadyExistsException"
, e);
}
catch (IOException e)
{
HandleRelaxedException("overwriting a dir with a file ", "FileAlreadyExistsException"
, e);
}
AssertIsDirectory(path);
AssertIsFile(child);
}
public virtual void TestCreateNewFile()
{
Describe("Foundational 'create a file' test");
Path path = Path("testCreateNewFile");
byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024 * 1024
, false);
ContractTestUtils.VerifyFileContents(GetFileSystem(), path, data);
}
public virtual void TestMkDirDepth1()
{
FileSystem fs = GetFileSystem();
Path dir = new Path("/testmkdirdepth1");
AssertPathDoesNotExist("directory already exists", dir);
fs.Mkdirs(dir);
ContractTestUtils.AssertIsDirectory(GetFileSystem(), dir);
AssertPathExists("directory already exists", dir);
AssertDeleted(dir, true);
}
public virtual void TestFsIsEncrypted()
{
Describe("create an empty file and call FileStatus.isEncrypted()");
Path path = Path("file");
ContractTestUtils.CreateFile(GetFileSystem(), path, false, new byte[0]);
FileStatus stat = GetFileSystem().GetFileStatus(path);
NUnit.Framework.Assert.IsFalse("Expecting false for stat.isEncrypted()", stat.IsEncrypted
());
}
public virtual void TestDeleteNonEmptyDirRecursive()
{
Path path = Path("testDeleteNonEmptyDirNonRecursive");
Mkdirs(path);
Path file = new Path(path, "childfile");
ContractTestUtils.WriteTextFile(GetFileSystem(), file, "goodbye, world", true);
AssertDeleted(path, true);
ContractTestUtils.AssertPathDoesNotExist(GetFileSystem(), "not deleted", file);
}
public virtual void TestRmEmptyRootDirNonRecursive()
{
//extra sanity checks here to avoid support calls about complete loss of data
SkipIfUnsupported(TestRootTestsEnabled);
Path root = new Path("/");
ContractTestUtils.AssertIsDirectory(GetFileSystem(), root);
bool deleted = GetFileSystem().Delete(root, true);
Log.Info("rm / of empty dir result is {}", deleted);
ContractTestUtils.AssertIsDirectory(GetFileSystem(), root);
}
public virtual void TestOpenReadZeroByteFile()
{
Describe("create & read a 0 byte file");
Path path = Path("zero.txt");
ContractTestUtils.Touch(GetFileSystem(), path);
instream = GetFileSystem().Open(path);
Assert.Equal(0, instream.GetPos());
//expect initial read to fail
int result = instream.Read();
AssertMinusOne("initial byte read", result);
}
public virtual void TestDeleteDeepEmptyDir()
{
Mkdirs(Path("testDeleteDeepEmptyDir/d1/d2/d3/d4"));
AssertDeleted(Path("testDeleteDeepEmptyDir/d1/d2/d3"), true);
FileSystem fs = GetFileSystem();
ContractTestUtils.AssertPathDoesNotExist(fs, "not deleted", Path("testDeleteDeepEmptyDir/d1/d2/d3/d4"
));
ContractTestUtils.AssertPathDoesNotExist(fs, "not deleted", Path("testDeleteDeepEmptyDir/d1/d2/d3"
));
ContractTestUtils.AssertPathExists(fs, "parent dir is deleted", Path("testDeleteDeepEmptyDir/d1/d2"
));
}
/// <exception cref="System.Exception"/>
public override void Setup()
{
base.Setup();
SkipIfUnsupported(SupportsSeek);
//delete the test directory
testPath = GetContract().GetTestPath();
smallSeekFile = Path("seekfile.txt");
zeroByteFile = Path("zero.txt");
byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255);
//this file now has a simple rule: offset => value
ContractTestUtils.CreateFile(GetFileSystem(), smallSeekFile, false, block);
ContractTestUtils.Touch(GetFileSystem(), zeroByteFile);
}
/// <exception cref="System.Exception"/>
public override void Setup()
{
base.Setup();
SkipIfUnsupported(SupportsConcat);
//delete the test directory
testPath = Path("test");
srcFile = new Path(testPath, "small.txt");
zeroByteFile = new Path(testPath, "zero.txt");
target = new Path(testPath, "target");
byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255);
ContractTestUtils.CreateFile(GetFileSystem(), srcFile, false, block);
ContractTestUtils.Touch(GetFileSystem(), zeroByteFile);
}
public virtual void TestAppendToExistingFile()
{
byte[] original = ContractTestUtils.Dataset(8192, 'A', 'Z');
byte[] appended = ContractTestUtils.Dataset(8192, '0', '9');
ContractTestUtils.CreateFile(GetFileSystem(), target, false, original);
FSDataOutputStream outputStream = GetFileSystem().Append(target);
outputStream.Write(appended);
outputStream.Close();
byte[] bytes = ContractTestUtils.ReadDataset(GetFileSystem(), target, original.Length
+ appended.Length);
ContractTestUtils.ValidateFileContent(bytes, new byte[][] { original, appended });
}
public virtual void TestRenameFileOverExistingFile()
{
Describe("Verify renaming a file onto an existing file matches expectations");
Path srcFile = Path("source-256.txt");
byte[] srcData = ContractTestUtils.Dataset(256, 'a', 'z');
ContractTestUtils.WriteDataset(GetFileSystem(), srcFile, srcData, srcData.Length,
1024, false);
Path destFile = Path("dest-512.txt");
byte[] destData = ContractTestUtils.Dataset(512, 'A', 'Z');
ContractTestUtils.WriteDataset(GetFileSystem(), destFile, destData, destData.Length
, 1024, false);
AssertIsFile(destFile);
bool renameOverwritesDest = IsSupported(RenameOverwritesDest);
bool renameReturnsFalseOnRenameDestExists = !IsSupported(RenameReturnsFalseIfDestExists
);
bool destUnchanged = true;
try
{
bool renamed = Rename(srcFile, destFile);
if (renameOverwritesDest)
{
// the filesystem supports rename(file, file2) by overwriting file2
Assert.True("Rename returned false", renamed);
destUnchanged = false;
}
else
{
// rename is rejected by returning 'false' or throwing an exception
if (renamed && !renameReturnsFalseOnRenameDestExists)
{
//expected an exception
string destDirLS = GenerateAndLogErrorListing(srcFile, destFile);
GetLog().Error("dest dir {}", destDirLS);
NUnit.Framework.Assert.Fail("expected rename(" + srcFile + ", " + destFile + " ) to fail,"
+ " but got success and destination of " + destDirLS);
}
}
}
catch (FileAlreadyExistsException e)
{
HandleExpectedException(e);
}
// verify that the destination file is as expected based on the expected
// outcome
ContractTestUtils.VerifyFileContents(GetFileSystem(), destFile, destUnchanged ? destData
: srcData);
}
public virtual void TestOverwriteExistingFile()
{
Describe("Overwrite an existing file and verify the new data is there");
Path path = Path("testOverwriteExistingFile");
byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
ContractTestUtils.WriteDataset(GetFileSystem(), path, data, data.Length, 1024, false
);
ContractTestUtils.VerifyFileContents(GetFileSystem(), path, data);
byte[] data2 = ContractTestUtils.Dataset(10 * 1024, 'A', 'Z');
ContractTestUtils.WriteDataset(GetFileSystem(), path, data2, data2.Length, 1024,
true);
ContractTestUtils.VerifyFileContents(GetFileSystem(), path, data2);
}
public virtual void TestDeleteSingleFile()
{
// Test delete of just a file
Path path = Path("testDeleteSingleFile/d1/d2");
Mkdirs(path);
Path file = new Path(path, "childfile");
ContractTestUtils.WriteTextFile(GetFileSystem(), file, "single file to be deleted."
, true);
ContractTestUtils.AssertPathExists(GetFileSystem(), "single file not created", file
);
AssertDeleted(file, false);
}
public virtual void TestConcatEmptyFiles()
{
ContractTestUtils.Touch(GetFileSystem(), target);
try
{
GetFileSystem().Concat(target, new Path[0]);
NUnit.Framework.Assert.Fail("expected a failure");
}
catch (Exception e)
{
//expected
HandleExpectedException(e);
}
}
public virtual void TestConcatOnSelf()
{
byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255);
ContractTestUtils.CreateFile(GetFileSystem(), target, false, block);
try
{
GetFileSystem().Concat(target, new Path[] { target });
}
catch (Exception e)
{
//expected
HandleExpectedException(e);
}
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual string GenerateAndLogErrorListing(Org.Apache.Hadoop.FS.Path
src, Org.Apache.Hadoop.FS.Path dst)
{
FileSystem fs = GetFileSystem();
GetLog().Error("src dir " + ContractTestUtils.Ls(fs, src.GetParent()));
string destDirLS = ContractTestUtils.Ls(fs, dst.GetParent());
if (fs.IsDirectory(dst))
{
//include the dir into the listing
destDirLS = destDirLS + "\n" + ContractTestUtils.Ls(fs, dst);
}
return(destDirLS);
}
public virtual void TestRenameNewFileSameDir()
{
Describe("rename a file into a new file in the same directory");
Path renameSrc = Path("rename_src");
Path renameTarget = Path("rename_dest");
byte[] data = ContractTestUtils.Dataset(256, 'a', 'z');
ContractTestUtils.WriteDataset(GetFileSystem(), renameSrc, data, data.Length, 1024
* 1024, false);
bool rename = Rename(renameSrc, renameTarget);
Assert.True("rename(" + renameSrc + ", " + renameTarget + ") returned false"
, rename);
ContractTestUtils.AssertListStatusFinds(GetFileSystem(), renameTarget.GetParent()
, renameTarget);
ContractTestUtils.VerifyFileContents(GetFileSystem(), renameTarget, data);
}
public virtual void TestRandomSeeks()
{
int limit = GetContract().GetLimit(TestRandomSeekCount, DefaultRandomSeekCount);
Describe("Testing " + limit + " random seeks");
int filesize = 10 * 1024;
byte[] buf = ContractTestUtils.Dataset(filesize, 0, 255);
Path randomSeekFile = Path("testrandomseeks.bin");
ContractTestUtils.CreateFile(GetFileSystem(), randomSeekFile, false, buf);
Random r = new Random();
FSDataInputStream stm = GetFileSystem().Open(randomSeekFile);
// Record the sequence of seeks and reads which trigger a failure.
int[] seeks = new int[10];
int[] reads = new int[10];
try
{
for (int i = 0; i < limit; i++)
{
int seekOff = r.Next(buf.Length);
int toRead = r.Next(Math.Min(buf.Length - seekOff, 32000));
seeks[i % seeks.Length] = seekOff;
reads[i % reads.Length] = toRead;
ContractTestUtils.VerifyRead(stm, buf, seekOff, toRead);
}
}
catch (Exception afe)
{
StringBuilder sb = new StringBuilder();
sb.Append("Sequence of actions:\n");
for (int j = 0; j < seeks.Length; j++)
{
sb.Append("seek @ ").Append(seeks[j]).Append(" ").Append("read ").Append(reads[j
]).Append("\n");
}
Log.Error(sb.ToString());
throw;
}
finally
{
stm.Close();
}
}