/// <summary>The src file is under FS, and the dst is on the local disk.</summary>
/// <remarks>
/// The src file is under FS, and the dst is on the local disk.
/// Copy it from FS control to the local dst name.
/// If src and dst are directories, the copyCrc parameter
/// determines whether to copy CRC files.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void CopyToLocalFile(Path src, Path dst, bool copyCrc)
{
if (!fs.IsDirectory(src))
{
// source is a file
fs.CopyToLocalFile(src, dst);
FileSystem localFs = GetLocal(GetConf()).GetRawFileSystem();
if (localFs.IsDirectory(dst))
{
dst = new Path(dst, src.GetName());
}
dst = GetChecksumFile(dst);
if (localFs.Exists(dst))
{
//remove old local checksum file
localFs.Delete(dst, true);
}
Path checksumFile = GetChecksumFile(src);
if (copyCrc && fs.Exists(checksumFile))
{
//copy checksum file
fs.CopyToLocalFile(checksumFile, dst);
}
}
else
{
FileStatus[] srcs = ListStatus(src);
foreach (FileStatus srcFile in srcs)
{
CopyToLocalFile(srcFile.GetPath(), new Path(dst, srcFile.GetPath().GetName()), copyCrc
);
}
}
}
/// <summary>Test Chmod 1.</summary>
/// <remarks>
/// Test Chmod 1. Create and write file on FS 2. Verify that exit code for
/// chmod on existing file is 0 3. Verify that exit code for chmod on
/// non-existing file is 1 4. Verify that exit code for chmod with glob input
/// on non-existing file is 1 5. Verify that exit code for chmod with glob
/// input on existing file in 0
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestChmod()
{
Path p1 = new Path(TestRootDir, "testChmod/fileExists");
string f1 = p1.ToUri().GetPath();
string f2 = new Path(TestRootDir, "testChmod/fileDoesNotExist").ToUri().GetPath();
string f3 = new Path(TestRootDir, "testChmod/nonExistingfiles*").ToUri().GetPath(
);
Path p4 = new Path(TestRootDir, "testChmod/file1");
Path p5 = new Path(TestRootDir, "testChmod/file2");
Path p6 = new Path(TestRootDir, "testChmod/file3");
string f7 = new Path(TestRootDir, "testChmod/file*").ToUri().GetPath();
// create and write test file
WriteFile(fileSys, p1);
Assert.True(fileSys.Exists(p1));
// Test 1: Test 1: exit code for chmod on existing is 0
string[] argv = new string[] { "-chmod", "777", f1 };
Assert.Equal(0, fsShell.Run(argv));
// Test 2: exit code for chmod on non-existing path is 1
string[] argv2 = new string[] { "-chmod", "777", f2 };
Assert.Equal(1, fsShell.Run(argv2));
// Test 3: exit code for chmod on non-existing path with globbed input is 1
string[] argv3 = new string[] { "-chmod", "777", f3 };
Assert.Equal(1, fsShell.Run(argv3));
// create required files
WriteFile(fileSys, p4);
Assert.True(fileSys.Exists(p4));
WriteFile(fileSys, p5);
Assert.True(fileSys.Exists(p5));
WriteFile(fileSys, p6);
Assert.True(fileSys.Exists(p6));
// Test 4: exit code for chmod on existing path with globbed input is 0
string[] argv4 = new string[] { "-chmod", "777", f7 };
Assert.Equal(0, fsShell.Run(argv4));
}
/// <summary>Rename files/dirs</summary>
/// <exception cref="System.IO.IOException"/>
public override bool Rename(Path src, Path dst)
{
if (fs.IsDirectory(src))
{
return(fs.Rename(src, dst));
}
else
{
if (fs.IsDirectory(dst))
{
dst = new Path(dst, src.GetName());
}
bool value = fs.Rename(src, dst);
if (!value)
{
return(false);
}
Path srcCheckFile = GetChecksumFile(src);
Path dstCheckFile = GetChecksumFile(dst);
if (fs.Exists(srcCheckFile))
{
//try to rename checksum
value = fs.Rename(srcCheckFile, dstCheckFile);
}
else
{
if (fs.Exists(dstCheckFile))
{
// no src checksum, so remove dst checksum
value = fs.Delete(dstCheckFile, true);
}
}
return(value);
}
}
/// <summary>
/// Implement the delete(Path, boolean) in checksum
/// file system.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public override bool Delete(Path f, bool recursive)
{
FileStatus fstatus = null;
try
{
fstatus = fs.GetFileStatus(f);
}
catch (FileNotFoundException)
{
return(false);
}
if (fstatus.IsDirectory())
{
//this works since the crcs are in the same
//directories and the files. so we just delete
//everything in the underlying filesystem
return(fs.Delete(f, recursive));
}
else
{
Path checkFile = GetChecksumFile(f);
if (fs.Exists(checkFile))
{
fs.Delete(checkFile, true);
}
return(fs.Delete(f, true));
}
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream CreateNonRecursive(Path f, FsPermission permission
, bool overwrite, int bufferSize, short replication, long blockSize, Progressable
progress)
{
return(Create(f, permission, overwrite, false, bufferSize, replication, blockSize
, progress));
}
/// <summary>Test Chgrp 1.</summary>
/// <remarks>
/// Test Chgrp 1. Create and write file on FS 2. Verify that exit code for
/// chgrp on existing file is 0 3. Verify that exit code for chgrp on
/// non-existing file is 1 4. Verify that exit code for chgrp with glob input
/// on non-existing file is 1 5. Verify that exit code for chgrp with glob
/// input on existing file in 0
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestChgrp()
{
Path p1 = new Path(TestRootDir, "testChgrp/fileExists");
string f1 = p1.ToUri().GetPath();
string f2 = new Path(TestRootDir, "testChgrp/fileDoesNotExist").ToUri().GetPath();
string f3 = new Path(TestRootDir, "testChgrp/nonExistingfiles*").ToUri().GetPath(
);
Path p4 = new Path(TestRootDir, "testChgrp/file1");
Path p5 = new Path(TestRootDir, "testChgrp/file2");
Path p6 = new Path(TestRootDir, "testChgrp/file3");
string f7 = new Path(TestRootDir, "testChgrp/file*").ToUri().GetPath();
// create and write test file
WriteFile(fileSys, p1);
Assert.True(fileSys.Exists(p1));
// Test 1: exit code for chgrp on existing file is 0
Change(0, null, "admin", f1);
// Test 2: exit code for chgrp on non existing path is 1
Change(1, null, "admin", f2);
Change(1, null, "admin", f2, f1);
// exit code used to be for last item
// Test 3: exit code for chgrp on non-existing path with globbed input is 1
Change(1, null, "admin", f3);
Change(1, null, "admin", f3, f1);
// create required files
WriteFile(fileSys, p4);
Assert.True(fileSys.Exists(p4));
WriteFile(fileSys, p5);
Assert.True(fileSys.Exists(p5));
WriteFile(fileSys, p6);
Assert.True(fileSys.Exists(p6));
// Test 4: exit code for chgrp on existing path with globbed input is 0
Change(0, null, "admin", f7);
Change(1, null, "admin", f2, f7);
}
/// <exception cref="System.Exception"/>
internal static void WriteFile(FileSystem fs, Path name)
{
FSDataOutputStream stm = fs.Create(name);
stm.WriteBytes("42\n");
stm.Close();
}
/// <exception cref="System.IO.IOException"/>
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
: base(file, fs.GetFileStatus(file).GetReplication())
{
this.datas = fs.GetRawFileSystem().Open(file, bufferSize);
this.fs = fs;
Path sumFile = fs.GetChecksumFile(file);
try
{
int sumBufferSize = fs.GetSumBufferSize(fs.GetBytesPerSum(), bufferSize);
sums = fs.GetRawFileSystem().Open(sumFile, sumBufferSize);
byte[] version = new byte[ChecksumVersion.Length];
sums.ReadFully(version);
if (!Arrays.Equals(version, ChecksumVersion))
{
throw new IOException("Not a checksum file: " + sumFile);
}
this.bytesPerSum = sums.ReadInt();
Set(fs.verifyChecksum, DataChecksum.NewCrc32(), bytesPerSum, 4);
}
catch (FileNotFoundException)
{
// quietly ignore
Set(fs.verifyChecksum, null, 1, 0);
}
catch (IOException e)
{
// loudly ignore
Log.Warn("Problem opening checksum file: " + file + ". Ignoring exception: ", e);
Set(fs.verifyChecksum, null, 1, 0);
}
}
/// <summary>Test Chown 1.</summary>
/// <remarks>
/// Test Chown 1. Create and write file on FS 2. Verify that exit code for
/// Chown on existing file is 0 3. Verify that exit code for Chown on
/// non-existing file is 1 4. Verify that exit code for Chown with glob input
/// on non-existing file is 1 5. Verify that exit code for Chown with glob
/// input on existing file in 0
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestChown()
{
Path p1 = new Path(TestRootDir, "testChown/fileExists");
string f1 = p1.ToUri().GetPath();
string f2 = new Path(TestRootDir, "testChown/fileDoesNotExist").ToUri().GetPath();
string f3 = new Path(TestRootDir, "testChown/nonExistingfiles*").ToUri().GetPath(
);
Path p4 = new Path(TestRootDir, "testChown/file1");
Path p5 = new Path(TestRootDir, "testChown/file2");
Path p6 = new Path(TestRootDir, "testChown/file3");
string f7 = new Path(TestRootDir, "testChown/file*").ToUri().GetPath();
// create and write test file
WriteFile(fileSys, p1);
Assert.True(fileSys.Exists(p1));
// Test 1: exit code for chown on existing file is 0
Change(0, "admin", null, f1);
// Test 2: exit code for chown on non-existing path is 1
Change(1, "admin", null, f2);
// Test 3: exit code for chown on non-existing path with globbed input is 1
Change(1, "admin", null, f3);
// create required files
WriteFile(fileSys, p4);
Assert.True(fileSys.Exists(p4));
WriteFile(fileSys, p5);
Assert.True(fileSys.Exists(p5));
WriteFile(fileSys, p6);
Assert.True(fileSys.Exists(p6));
// Test 4: exit code for chown on existing path with globbed input is 0
Change(0, "admin", null, f7);
//Test 5: test for setOwner invocation on FS from command handler.
Change(0, "admin", "Test", f1);
Change(0, "admin", string.Empty, f1);
}
/// <summary>
/// Moves files to a bad file directory on the same device, so that their
/// storage will not be reused.
/// </summary>
public override bool ReportChecksumFailure(Path p, FSDataInputStream @in, long inPos
, FSDataInputStream sums, long sumsPos)
{
try
{
// canonicalize f
FilePath f = ((RawLocalFileSystem)fs).PathToFile(p).GetCanonicalFile();
// find highest writable parent dir of f on the same device
string device = new DF(f, GetConf()).GetMount();
FilePath parent = f.GetParentFile();
FilePath dir = null;
while (parent != null && FileUtil.CanWrite(parent) && parent.ToString().StartsWith
(device))
{
dir = parent;
parent = parent.GetParentFile();
}
if (dir == null)
{
throw new IOException("not able to find the highest writable parent dir");
}
// move the file there
FilePath badDir = new FilePath(dir, "bad_files");
if (!badDir.Mkdirs())
{
if (!badDir.IsDirectory())
{
throw new IOException("Mkdirs failed to create " + badDir.ToString());
}
}
string suffix = "." + rand.Next();
FilePath badFile = new FilePath(badDir, f.GetName() + suffix);
Log.Warn("Moving bad file " + f + " to " + badFile);
@in.Close();
// close it first
bool b = f.RenameTo(badFile);
// rename it
if (!b)
{
Log.Warn("Ignoring failure of renameTo");
}
// move checksum file too
FilePath checkFile = ((RawLocalFileSystem)fs).PathToFile(GetChecksumFile(p));
// close the stream before rename to release the file handle
sums.Close();
b = checkFile.RenameTo(new FilePath(badDir, checkFile.GetName() + suffix));
if (!b)
{
Log.Warn("Ignoring failure of renameTo");
}
}
catch (IOException e)
{
Log.Warn("Error moving bad file " + p + ": " + e);
}
return(false);
}
/// <exception cref="System.IO.IOException"/>
public override void SetOwner(Path p, string username, string groupname)
{
string f = MakeQualified(p).ToString();
if (username != null)
{
owners[f] = username;
}
if (groupname != null)
{
groups[f] = groupname;
}
}
/// <exception cref="System.IO.IOException"/>
public TextRecordInputStream(Display _enclosing, FileStatus f)
{
this._enclosing = _enclosing;
Path fpath = f.GetPath();
Configuration lconf = this._enclosing.GetConf();
this.r = new SequenceFile.Reader(lconf, SequenceFile.Reader.File(fpath));
this.key = ReflectionUtils.NewInstance(this.r.GetKeyClass().AsSubclass <IWritableComparable <> >(), lconf);
this.val = ReflectionUtils.NewInstance(this.r.GetValueClass().AsSubclass <IWritable
>(), lconf);
this.inbuf = new DataInputBuffer();
this.outbuf = new DataOutputBuffer();
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual string GenerateAndLogErrorListing(Org.Apache.Hadoop.FS.Path
src, Org.Apache.Hadoop.FS.Path dst)
{
FileSystem fs = GetFileSystem();
GetLog().Error("src dir " + ContractTestUtils.Ls(fs, src.GetParent()));
string destDirLS = ContractTestUtils.Ls(fs, dst.GetParent());
if (fs.IsDirectory(dst))
{
//include the dir into the listing
destDirLS = destDirLS + "\n" + ContractTestUtils.Ls(fs, dst);
}
return(destDirLS);
}
/// <summary>Set replication for an existing file.</summary>
/// <remarks>
/// Set replication for an existing file.
/// Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt>
/// </remarks>
/// <param name="src">file name</param>
/// <param name="replication">new replication</param>
/// <exception cref="System.IO.IOException"/>
/// <returns>
/// true if successful;
/// false if file does not exist or is a directory
/// </returns>
public override bool SetReplication(Path src, short replication)
{
bool value = fs.SetReplication(src, replication);
if (!value)
{
return(false);
}
Path checkFile = GetChecksumFile(src);
if (Exists(checkFile))
{
fs.SetReplication(checkFile, replication);
}
return(true);
}
/// <exception cref="System.IO.IOException"/>
public ChecksumFSOutputSummer(ChecksumFileSystem fs, Path file, bool overwrite, int
bufferSize, short replication, long blockSize, Progressable progress, FsPermission
permission)
: base(DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, fs.GetBytesPerSum())
)
{
int bytesPerSum = fs.GetBytesPerSum();
this.datas = fs.GetRawFileSystem().Create(file, permission, overwrite, bufferSize
, replication, blockSize, progress);
int sumBufferSize = fs.GetSumBufferSize(bytesPerSum, bufferSize);
this.sums = fs.GetRawFileSystem().Create(fs.GetChecksumFile(file), permission, true
, sumBufferSize, replication, blockSize, null);
sums.Write(ChecksumVersion, 0, ChecksumVersion.Length);
sums.WriteInt(bytesPerSum);
}
/// <summary>Opens an FSDataInputStream at the indicated Path.</summary>
/// <param name="f">the file name to open</param>
/// <param name="bufferSize">the size of the buffer to be used.</param>
/// <exception cref="System.IO.IOException"/>
public override FSDataInputStream Open(Path f, int bufferSize)
{
FileSystem fs;
InputStream @in;
if (verifyChecksum)
{
fs = this;
@in = new ChecksumFileSystem.ChecksumFSInputChecker(this, f, bufferSize);
}
else
{
fs = GetRawFileSystem();
@in = fs.Open(f, bufferSize);
}
return(new ChecksumFileSystem.FSDataBoundedInputStream(fs, f, @in));
}
/// <exception cref="System.IO.IOException"/>
private FSDataOutputStream Create(Path f, FsPermission permission, bool overwrite
, bool createParent, int bufferSize, short replication, long blockSize, Progressable
progress)
{
Path parent = f.GetParent();
if (parent != null)
{
if (!createParent && !Exists(parent))
{
throw new FileNotFoundException("Parent directory doesn't exist: " + parent);
}
else
{
if (!Mkdirs(parent))
{
throw new IOException("Mkdirs failed to create " + parent + " (exists=" + Exists(
parent) + ", cwd=" + GetWorkingDirectory() + ")");
}
}
}
FSDataOutputStream @out;
if (writeChecksum)
{
@out = new FSDataOutputStream(new ChecksumFileSystem.ChecksumFSOutputSummer(this,
f, overwrite, bufferSize, replication, blockSize, progress, permission), null);
}
else
{
@out = fs.Create(f, permission, overwrite, bufferSize, replication, blockSize, progress
);
// remove the checksum file since we aren't writing one
Path checkFile = GetChecksumFile(f);
if (fs.Exists(checkFile))
{
fs.Delete(checkFile, true);
}
}
return(@out);
}
public virtual void Setup()
{
contract = CreateContract(CreateConfiguration());
contract.Init();
//skip tests if they aren't enabled
AssumeEnabled();
//extract the test FS
fileSystem = contract.GetTestFileSystem();
NUnit.Framework.Assert.IsNotNull("null filesystem", fileSystem);
URI fsURI = fileSystem.GetUri();
Log.Info("Test filesystem = {} implemented by {}", fsURI, fileSystem);
//sanity check to make sure that the test FS picked up really matches
//the scheme chosen. This is to avoid defaulting back to the localFS
//which would be drastic for root FS tests
Assert.Equal("wrong filesystem of " + fsURI, contract.GetScheme
(), fsURI.GetScheme());
//create the test path
testPath = GetContract().GetTestPath();
Mkdirs(testPath);
}
/// <exception cref="System.Exception"/>
public virtual void TestGetWithInvalidSourcePathShouldNotDisplayNullInConsole()
{
Configuration conf = new Configuration();
FsShell shell = new FsShell();
shell.SetConf(conf);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
TextWriter @out = new TextWriter(bytes);
TextWriter oldErr = System.Console.Error;
Runtime.SetErr(@out);
string results;
try
{
Path tdir = new Path(TestRootDir, "notNullCopy");
fileSys.Delete(tdir, true);
fileSys.Mkdirs(tdir);
string[] args = new string[3];
args[0] = "-get";
args[1] = new Path(tdir.ToUri().GetPath(), "/invalidSrc").ToString();
args[2] = new Path(tdir.ToUri().GetPath(), "/invalidDst").ToString();
Assert.True("file exists", !fileSys.Exists(new Path(args[1])));
Assert.True("file exists", !fileSys.Exists(new Path(args[2])));
int run = shell.Run(args);
results = bytes.ToString();
Assert.Equal("Return code should be 1", 1, run);
Assert.True(" Null is coming when source path is invalid. ", !results
.Contains("get: null"));
Assert.True(" Not displaying the intended message ", results.Contains
("get: `" + args[1] + "': No such file or directory"));
}
finally
{
IOUtils.CloseStream(@out);
Runtime.SetErr(oldErr);
}
}
/// <exception cref="System.IO.IOException"/>
public override FileStatus GetFileStatus(Path p)
{
string f = MakeQualified(p).ToString();
FileStatus stat = base.GetFileStatus(p);
stat.GetPermission();
if (owners.Contains(f))
{
stat.SetOwner("STUB-" + owners[f]);
}
else
{
stat.SetOwner("REAL-" + stat.GetOwner());
}
if (groups.Contains(f))
{
stat.SetGroup("STUB-" + groups[f]);
}
else
{
stat.SetGroup("REAL-" + stat.GetGroup());
}
return(stat);
}
/// <exception cref="System.IO.IOException"/>
internal virtual bool Rename(Org.Apache.Hadoop.FS.Path src, Org.Apache.Hadoop.FS.Path
dst)
{
return(GetFileSystem().Rename(src, dst));
}
/// <summary>Assert that a delete succeeded</summary>
/// <param name="path">path to delete</param>
/// <param name="recursive">recursive flag</param>
/// <exception cref="System.IO.IOException">IO problems</exception>
protected internal virtual void AssertDeleted(Org.Apache.Hadoop.FS.Path path, bool
recursive)
{
ContractTestUtils.AssertDeleted(fileSystem, path, recursive);
}
/// <summary>
/// Assert that a file exists and whose
/// <see cref="Org.Apache.Hadoop.FS.FileStatus"/>
/// entry
/// declares that this is a file and not a symlink or directory.
/// </summary>
/// <exception cref="System.IO.IOException">IO problems during file operations</exception>
protected internal virtual void Mkdirs(Org.Apache.Hadoop.FS.Path path)
{
Assert.True("Failed to mkdir " + path, fileSystem.Mkdirs(path));
}
/// <summary>
/// Assert that a file exists and whose
/// <see cref="Org.Apache.Hadoop.FS.FileStatus"/>
/// entry
/// declares that this is a file and not a symlink or directory.
/// </summary>
/// <param name="path">name of the file</param>
/// <exception cref="System.IO.IOException">IO problems during file operations</exception>
protected internal virtual void AssertIsDirectory(Org.Apache.Hadoop.FS.Path path)
{
ContractTestUtils.AssertIsDirectory(fileSystem, path);
}
/// <summary>
/// Assert that a file exists and whose
/// <see cref="Org.Apache.Hadoop.FS.FileStatus"/>
/// entry
/// declares that this is a file and not a symlink or directory.
/// </summary>
/// <param name="filename">name of the file</param>
/// <exception cref="System.IO.IOException">IO problems during file operations</exception>
protected internal virtual void AssertIsFile(Org.Apache.Hadoop.FS.Path filename)
{
ContractTestUtils.AssertIsFile(fileSystem, filename);
}
/// <summary>assert that a path does not</summary>
/// <param name="message">message to use in an assertion</param>
/// <param name="path">path to probe</param>
/// <exception cref="System.IO.IOException">IO problems</exception>
public virtual void AssertPathDoesNotExist(string message, Org.Apache.Hadoop.FS.Path
path)
{
ContractTestUtils.AssertPathDoesNotExist(fileSystem, message, path);
}
/// <summary>List a path in the test FS</summary>
/// <param name="path">path to list</param>
/// <returns>the contents of the path/dir</returns>
/// <exception cref="System.IO.IOException">IO problems</exception>
protected internal virtual string Ls(Org.Apache.Hadoop.FS.Path path)
{
return(ContractTestUtils.Ls(fileSystem, path));
}
/// <exception cref="System.IO.IOException"/>
public override void CopyToLocalFile(bool delSrc, Path src, Path dst)
{
FileUtil.Copy(this, src, this, dst, delSrc, GetConf());
}
/// <summary>Convert a path to a File.</summary>
public virtual FilePath PathToFile(Path path)
{
return(((RawLocalFileSystem)fs).PathToFile(path));
}
/// <exception cref="System.IO.IOException"/>
public override Path GetLinkTarget(Path f)
{
return(fs.GetLinkTarget(f));
}