/// <exception cref="System.IO.IOException"/>
private void OneTest(int offBegin, int offEnd, FileStatus status)
{
if (offBegin > offEnd)
{
int tmp = offBegin;
offBegin = offEnd;
offEnd = tmp;
}
BlockLocation[] locations = fs.GetFileBlockLocations(status, offBegin, offEnd - offBegin
);
if (offBegin < status.GetLen())
{
Arrays.Sort(locations, new _IComparer_69());
offBegin = (int)Math.Min(offBegin, status.GetLen() - 1);
offEnd = (int)Math.Min(offEnd, status.GetLen());
BlockLocation first = locations[0];
BlockLocation last = locations[locations.Length - 1];
Assert.True(first.GetOffset() <= offBegin);
Assert.True(offEnd <= last.GetOffset() + last.GetLength());
}
else
{
Assert.True(locations.Length == 0);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetFileBlockLocations2()
{
FileStatus status = fs.GetFileStatus(path);
for (int i = 0; i < 1000; ++i)
{
int offBegin = random.Next((int)(2 * status.GetLen()));
int offEnd = random.Next((int)(2 * status.GetLen()));
OneTest(offBegin, offEnd, status);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetFileBlockLocations1()
{
FileStatus status = fs.GetFileStatus(path);
OneTest(0, (int)status.GetLen(), status);
OneTest(0, (int)status.GetLen() * 2, status);
OneTest((int)status.GetLen() * 2, (int)status.GetLen() * 4, status);
OneTest((int)status.GetLen() / 2, (int)status.GetLen() * 3, status);
OneTest((int)status.GetLen(), (int)status.GetLen() * 2, status);
for (int i = 0; i < 10; ++i)
{
OneTest((int)status.GetLen() * i / 10, (int)status.GetLen() * (i + 1) / 10, status
);
}
}
/// <exception cref="System.IO.IOException"/>
private void ListSubtree(FileStatus rootStatus, SequenceFile.Writer writer)
{
Path rootFile = rootStatus.GetPath();
if (rootStatus.IsFile())
{
nrFiles++;
// For a regular file generate <fName,offset> pairs
long blockSize = fs.GetDefaultBlockSize(rootFile);
long fileLength = rootStatus.GetLen();
for (long offset = 0; offset < fileLength; offset += blockSize)
{
writer.Append(new Text(rootFile.ToString()), new LongWritable(offset));
}
return;
}
FileStatus[] children = null;
try
{
children = fs.ListStatus(rootFile);
}
catch (FileNotFoundException)
{
throw new IOException("Could not get listing for " + rootFile);
}
for (int i = 0; i < children.Length; i++)
{
ListSubtree(children[i], writer);
}
}
public virtual void TestTruncate()
{
short repl = 3;
int blockSize = 1024;
int numOfBlocks = 2;
DistributedFileSystem fs = cluster.GetFileSystem();
Path dir = GetTestRootPath(fc, "test/hadoop");
Path file = GetTestRootPath(fc, "test/hadoop/file");
byte[] data = FileSystemTestHelper.GetFileData(numOfBlocks, blockSize);
FileSystemTestHelper.CreateFile(fs, file, data, blockSize, repl);
int newLength = blockSize;
bool isReady = fc.Truncate(file, newLength);
NUnit.Framework.Assert.IsTrue("Recovery is not expected.", isReady);
FileStatus fileStatus = fc.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), newLength);
AppendTestUtil.CheckFullFile(fs, file, newLength, data, file.ToString());
ContentSummary cs = fs.GetContentSummary(dir);
NUnit.Framework.Assert.AreEqual("Bad disk space usage", cs.GetSpaceConsumed(), newLength
* repl);
NUnit.Framework.Assert.IsTrue(fs.Delete(dir, true));
}
/// <summary>Validates the toString method for FileStatus.</summary>
/// <param name="fileStatus">FileStatus to be validated</param>
/// <exception cref="System.IO.IOException"/>
private void ValidateToString(FileStatus fileStatus)
{
StringBuilder expected = new StringBuilder();
expected.Append("FileStatus{");
expected.Append("path=").Append(fileStatus.GetPath()).Append("; ");
expected.Append("isDirectory=").Append(fileStatus.IsDirectory()).Append("; ");
if (!fileStatus.IsDirectory())
{
expected.Append("length=").Append(fileStatus.GetLen()).Append("; ");
expected.Append("replication=").Append(fileStatus.GetReplication()).Append("; ");
expected.Append("blocksize=").Append(fileStatus.GetBlockSize()).Append("; ");
}
expected.Append("modification_time=").Append(fileStatus.GetModificationTime()).Append
("; ");
expected.Append("access_time=").Append(fileStatus.GetAccessTime()).Append("; ");
expected.Append("owner=").Append(fileStatus.GetOwner()).Append("; ");
expected.Append("group=").Append(fileStatus.GetGroup()).Append("; ");
expected.Append("permission=").Append(fileStatus.GetPermission()).Append("; ");
if (fileStatus.IsSymlink())
{
expected.Append("isSymlink=").Append(true).Append("; ");
expected.Append("symlink=").Append(fileStatus.GetSymlink()).Append("}");
}
else
{
expected.Append("isSymlink=").Append(false).Append("}");
}
Assert.Equal(expected.ToString(), fileStatus.ToString());
}
/// <exception cref="System.IO.IOException"/>
internal override long DoIO(Reporter reporter, string name, long newLength)
{
// IOMapperBase
// in bytes
bool isClosed = fs.Truncate(filePath, newLength);
reporter.SetStatus("truncating " + name + " to newLength " + newLength + " ::host = "
+ hostName);
for (int i = 0; !isClosed; i++)
{
try
{
Sharpen.Thread.Sleep(Delay);
}
catch (Exception)
{
}
FileStatus status = fs.GetFileStatus(filePath);
System.Diagnostics.Debug.Assert(status != null, "status is null");
isClosed = (status.GetLen() == newLength);
reporter.SetStatus("truncate recover for " + name + " to newLength " + newLength
+ " attempt " + i + " ::host = " + hostName);
}
return(Sharpen.Extensions.ValueOf(fileSize - newLength));
}
/// <summary>
/// Construct given a
/// <see cref="FileContext"/>
/// and a
/// <see cref="Path"/>
/// .
/// </summary>
/// <exception cref="System.IO.IOException"/>
public AvroFSInput(FileContext fc, Path p)
{
FileStatus status = fc.GetFileStatus(p);
this.len = status.GetLen();
this.stream = fc.Open(p);
}
private FileStatus DeprecatedGetFileLinkStatusInternal(Path f)
{
string target = FileUtil.ReadLink(new FilePath(f.ToString()));
try
{
FileStatus fs = GetFileStatus(f);
// If f refers to a regular file or directory
if (target.IsEmpty())
{
return(fs);
}
// Otherwise f refers to a symlink
return(new FileStatus(fs.GetLen(), false, fs.GetReplication(), fs.GetBlockSize(),
fs.GetModificationTime(), fs.GetAccessTime(), fs.GetPermission(), fs.GetOwner(),
fs.GetGroup(), new Path(target), f));
}
catch (FileNotFoundException e)
{
/* The exists method in the File class returns false for dangling
* links so we can get a FileNotFoundException for links that exist.
* It's also possible that we raced with a delete of the link. Use
* the readBasicFileAttributes method in java.nio.file.attributes
* when available.
*/
if (!target.IsEmpty())
{
return(new FileStatus(0, false, 0, 0, 0, 0, FsPermission.GetDefault(), string.Empty
, string.Empty, new Path(target), f));
}
// f refers to a file or directory that does not exist
throw;
}
}
/// <exception cref="System.IO.IOException"/>
public override bool Truncate(Path f, long newLength)
{
FileStatus status = GetFileStatus(f);
if (status == null)
{
throw new FileNotFoundException("File " + f + " not found");
}
if (status.IsDirectory())
{
throw new IOException("Cannot truncate a directory (=" + f + ")");
}
long oldLength = status.GetLen();
if (newLength > oldLength)
{
throw new ArgumentException("Cannot truncate to a larger file size. Current size: "
+ oldLength + ", truncate size: " + newLength + ".");
}
using (FileOutputStream @out = new FileOutputStream(PathToFile(f), true))
{
try
{
@out.GetChannel().Truncate(newLength);
}
catch (IOException e)
{
throw new FSError(e);
}
}
return(true);
}
/// <summary>Constructor</summary>
/// <param name="stat">a file status</param>
/// <param name="locations">a file's block locations</param>
/// <exception cref="System.IO.IOException"/>
public LocatedFileStatus(FileStatus stat, BlockLocation[] locations)
: this(stat.GetLen(), stat.IsDirectory(), stat.GetReplication(), stat.GetBlockSize
(), stat.GetModificationTime(), stat.GetAccessTime(), stat.GetPermission(), stat
.GetOwner(), stat.GetGroup(), null, stat.GetPath(), locations)
{
if (stat.IsSymlink())
{
SetSymlink(stat.GetSymlink());
}
}
/// <exception cref="System.IO.IOException"/>
private string ReadFile(string @out)
{
Path path = new Path(@out);
FileStatus stat = lfs.GetFileStatus(path);
FSDataInputStream @in = lfs.Open(path);
byte[] buffer = new byte[(int)stat.GetLen()];
@in.ReadFully(buffer);
@in.Close();
lfs.Delete(path, false);
return(Runtime.GetStringForBytes(buffer));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestCreateFileAndMkdirs()
{
Path test_dir = new Path(TestRootDir, "test_dir");
Path test_file = new Path(test_dir, "file1");
Assert.True(fileSys.Mkdirs(test_dir));
int fileSize = new Random().Next(1 << 20) + 1;
FileSystemTestHelper.WriteFile(fileSys, test_file, fileSize);
{
//check FileStatus and ContentSummary
FileStatus status = fileSys.GetFileStatus(test_file);
Assert.Equal(fileSize, status.GetLen());
ContentSummary summary = fileSys.GetContentSummary(test_dir);
Assert.Equal(fileSize, summary.GetLength());
}
// creating dir over a file
Path bad_dir = new Path(test_file, "another_dir");
try
{
fileSys.Mkdirs(bad_dir);
NUnit.Framework.Assert.Fail("Failed to detect existing file in path");
}
catch (ParentNotDirectoryException)
{
}
// Expected
try
{
fileSys.Mkdirs(null);
NUnit.Framework.Assert.Fail("Failed to detect null in mkdir arg");
}
catch (ArgumentException)
{
}
}
/// <summary>Validate the accessors for FileStatus.</summary>
/// <param name="fileStatus">FileStatus to checked</param>
/// <param name="length">expected length</param>
/// <param name="isdir">expected isDirectory</param>
/// <param name="replication">expected replication</param>
/// <param name="blocksize">expected blocksize</param>
/// <param name="mtime">expected modification time</param>
/// <param name="atime">expected access time</param>
/// <param name="permission">expected permission</param>
/// <param name="owner">expected owner</param>
/// <param name="group">expected group</param>
/// <param name="symlink">expected symlink</param>
/// <param name="path">expected path</param>
/// <exception cref="System.IO.IOException"/>
private void ValidateAccessors(FileStatus fileStatus, long length, bool isdir, int
replication, long blocksize, long mtime, long atime, FsPermission permission, string
owner, string group, Path symlink, Path path)
{
Assert.Equal(length, fileStatus.GetLen());
Assert.Equal(isdir, fileStatus.IsDirectory());
Assert.Equal(replication, fileStatus.GetReplication());
Assert.Equal(blocksize, fileStatus.GetBlockSize());
Assert.Equal(mtime, fileStatus.GetModificationTime());
Assert.Equal(atime, fileStatus.GetAccessTime());
Assert.Equal(permission, fileStatus.GetPermission());
Assert.Equal(owner, fileStatus.GetOwner());
Assert.Equal(group, fileStatus.GetGroup());
if (symlink == null)
{
NUnit.Framework.Assert.IsFalse(fileStatus.IsSymlink());
}
else
{
Assert.True(fileStatus.IsSymlink());
Assert.Equal(symlink, fileStatus.GetSymlink());
}
Assert.Equal(path, fileStatus.GetPath());
}
