public virtual void TestSymbolicLinks()
{
WriteFile(fc, File1, FileLen);
WriteFile(fc, File2, FileLen);
WriteFile(fc, File3, FileLen);
Path dir4 = new Path(TestDir, "dir4");
Path dir5 = new Path(dir4, "dir5");
Path file4 = new Path(dir4, "file4");
fc.CreateSymlink(Dir1, dir5, true);
fc.CreateSymlink(File1, file4, true);
RemoteIterator <LocatedFileStatus> itor = fc.Util().ListFiles(dir4, true);
LocatedFileStatus stat = itor.Next();
NUnit.Framework.Assert.IsTrue(stat.IsFile());
NUnit.Framework.Assert.AreEqual(fc.MakeQualified(File2), stat.GetPath());
stat = itor.Next();
NUnit.Framework.Assert.IsTrue(stat.IsFile());
NUnit.Framework.Assert.AreEqual(fc.MakeQualified(File3), stat.GetPath());
stat = itor.Next();
NUnit.Framework.Assert.IsTrue(stat.IsFile());
NUnit.Framework.Assert.AreEqual(fc.MakeQualified(File1), stat.GetPath());
NUnit.Framework.Assert.IsFalse(itor.HasNext());
itor = fc.Util().ListFiles(dir4, false);
stat = itor.Next();
NUnit.Framework.Assert.IsTrue(stat.IsFile());
NUnit.Framework.Assert.AreEqual(fc.MakeQualified(File1), stat.GetPath());
NUnit.Framework.Assert.IsFalse(itor.HasNext());
}
/// <exception cref="System.IO.IOException"/>
private IList <FileStatus> SingleThreadedListStatus(JobContext job, Path[] dirs, PathFilter
inputFilter, bool recursive)
{
IList <FileStatus> result = new AList <FileStatus>();
IList <IOException> errors = new AList <IOException>();
for (int i = 0; i < dirs.Length; ++i)
{
Path p = dirs[i];
FileSystem fs = p.GetFileSystem(job.GetConfiguration());
FileStatus[] matches = fs.GlobStatus(p, inputFilter);
if (matches == null)
{
errors.AddItem(new IOException("Input path does not exist: " + p));
}
else
{
if (matches.Length == 0)
{
errors.AddItem(new IOException("Input Pattern " + p + " matches 0 files"));
}
else
{
foreach (FileStatus globStat in matches)
{
if (globStat.IsDirectory())
{
RemoteIterator <LocatedFileStatus> iter = fs.ListLocatedStatus(globStat.GetPath());
while (iter.HasNext())
{
LocatedFileStatus stat = iter.Next();
if (inputFilter.Accept(stat.GetPath()))
{
if (recursive && stat.IsDirectory())
{
AddInputPathRecursively(result, fs, stat.GetPath(), inputFilter);
}
else
{
result.AddItem(stat);
}
}
}
}
else
{
result.AddItem(globStat);
}
}
}
}
}
if (!errors.IsEmpty())
{
throw new InvalidInputException(errors);
}
return(result);
}
/// <exception cref="System.Exception"/>
public virtual LocatedFileStatusFetcher.ProcessInputDirCallable.Result Call()
{
LocatedFileStatusFetcher.ProcessInputDirCallable.Result result = new LocatedFileStatusFetcher.ProcessInputDirCallable.Result
();
result.fs = fs;
if (fileStatus.IsDirectory())
{
RemoteIterator <LocatedFileStatus> iter = fs.ListLocatedStatus(fileStatus.GetPath(
));
while (iter.HasNext())
{
LocatedFileStatus stat = iter.Next();
if (inputFilter.Accept(stat.GetPath()))
{
if (recursive && stat.IsDirectory())
{
result.dirsNeedingRecursiveCalls.AddItem(stat);
}
else
{
result.locatedFileStatuses.AddItem(stat);
}
}
}
}
else
{
result.locatedFileStatuses.AddItem(fileStatus);
}
return(result);
}
public virtual void TestFile()
{
fc.Mkdir(TestDir, FsPermission.GetDefault(), true);
WriteFile(fc, File1, FileLen);
RemoteIterator <LocatedFileStatus> itor = fc.Util().ListFiles(File1, true);
LocatedFileStatus stat = itor.Next();
NUnit.Framework.Assert.IsFalse(itor.HasNext());
NUnit.Framework.Assert.IsTrue(stat.IsFile());
NUnit.Framework.Assert.AreEqual(FileLen, stat.GetLen());
NUnit.Framework.Assert.AreEqual(fc.MakeQualified(File1), stat.GetPath());
NUnit.Framework.Assert.AreEqual(1, stat.GetBlockLocations().Length);
itor = fc.Util().ListFiles(File1, false);
stat = itor.Next();
NUnit.Framework.Assert.IsFalse(itor.HasNext());
NUnit.Framework.Assert.IsTrue(stat.IsFile());
NUnit.Framework.Assert.AreEqual(FileLen, stat.GetLen());
NUnit.Framework.Assert.AreEqual(fc.MakeQualified(File1), stat.GetPath());
NUnit.Framework.Assert.AreEqual(1, stat.GetBlockLocations().Length);
}
/// <summary>Add files in the input path recursively into the results.</summary>
/// <param name="result">The List to store all files.</param>
/// <param name="fs">The FileSystem.</param>
/// <param name="path">The input path.</param>
/// <param name="inputFilter">The input filter that can be used to filter files/dirs.
/// </param>
/// <exception cref="System.IO.IOException"/>
protected internal virtual void AddInputPathRecursively(IList <FileStatus> result,
FileSystem fs, Path path, PathFilter inputFilter)
{
RemoteIterator <LocatedFileStatus> iter = fs.ListLocatedStatus(path);
while (iter.HasNext())
{
LocatedFileStatus stat = iter.Next();
if (inputFilter.Accept(stat.GetPath()))
{
if (stat.IsDirectory())
{
AddInputPathRecursively(result, fs, stat.GetPath(), inputFilter);
}
else
{
result.AddItem(stat);
}
}
}
}
public virtual void TestDirectory()
{
fc.Mkdir(Dir1, FsPermission.GetDefault(), true);
// test empty directory
RemoteIterator <LocatedFileStatus> itor = fc.Util().ListFiles(Dir1, true);
NUnit.Framework.Assert.IsFalse(itor.HasNext());
itor = fc.Util().ListFiles(Dir1, false);
NUnit.Framework.Assert.IsFalse(itor.HasNext());
// testing directory with 1 file
WriteFile(fc, File2, FileLen);
itor = fc.Util().ListFiles(Dir1, true);
LocatedFileStatus stat = itor.Next();
NUnit.Framework.Assert.IsFalse(itor.HasNext());
NUnit.Framework.Assert.IsTrue(stat.IsFile());
NUnit.Framework.Assert.AreEqual(FileLen, stat.GetLen());
NUnit.Framework.Assert.AreEqual(fc.MakeQualified(File2), stat.GetPath());
NUnit.Framework.Assert.AreEqual(1, stat.GetBlockLocations().Length);
itor = fc.Util().ListFiles(Dir1, false);
stat = itor.Next();
NUnit.Framework.Assert.IsFalse(itor.HasNext());
NUnit.Framework.Assert.IsTrue(stat.IsFile());
NUnit.Framework.Assert.AreEqual(FileLen, stat.GetLen());
NUnit.Framework.Assert.AreEqual(fc.MakeQualified(File2), stat.GetPath());
NUnit.Framework.Assert.AreEqual(1, stat.GetBlockLocations().Length);
// test more complicated directory
WriteFile(fc, File1, FileLen);
WriteFile(fc, File3, FileLen);
itor = fc.Util().ListFiles(TestDir, true);
stat = itor.Next();
NUnit.Framework.Assert.IsTrue(stat.IsFile());
NUnit.Framework.Assert.AreEqual(fc.MakeQualified(File2), stat.GetPath());
stat = itor.Next();
NUnit.Framework.Assert.IsTrue(stat.IsFile());
NUnit.Framework.Assert.AreEqual(fc.MakeQualified(File3), stat.GetPath());
stat = itor.Next();
NUnit.Framework.Assert.IsTrue(stat.IsFile());
NUnit.Framework.Assert.AreEqual(fc.MakeQualified(File1), stat.GetPath());
NUnit.Framework.Assert.IsFalse(itor.HasNext());
itor = fc.Util().ListFiles(TestDir, false);
stat = itor.Next();
NUnit.Framework.Assert.IsTrue(stat.IsFile());
NUnit.Framework.Assert.AreEqual(fc.MakeQualified(File1), stat.GetPath());
NUnit.Framework.Assert.IsFalse(itor.HasNext());
}