public void CaseSensitivityTest()
{
Modification m = ModificationMother.CreateModification("x.xml", "/working/sources");
PathFilter filter = new PathFilter();
filter.Pattern = "**/*.xml";
Assert.IsTrue(filter.Accept(m));
m.FileName = "test.Xml";
Assert.IsFalse(filter.Accept(m));
filter.CaseSensitive = false;
Assert.IsTrue(filter.Accept(m));
}
public void ShouldNotAcceptModificationsWithNullFolder()
{
Modification modification = new Modification();
PathFilter filter = new PathFilter();
filter.Pattern = "*.*";
Assert.IsFalse(filter.Accept(modification));
}
/// <exception cref="System.IO.IOException"/>
private IList <FileStatus> SingleThreadedListStatus(JobContext job, Path[] dirs, PathFilter
inputFilter, bool recursive)
{
IList <FileStatus> result = new AList <FileStatus>();
IList <IOException> errors = new AList <IOException>();
for (int i = 0; i < dirs.Length; ++i)
{
Path p = dirs[i];
FileSystem fs = p.GetFileSystem(job.GetConfiguration());
FileStatus[] matches = fs.GlobStatus(p, inputFilter);
if (matches == null)
{
errors.AddItem(new IOException("Input path does not exist: " + p));
}
else
{
if (matches.Length == 0)
{
errors.AddItem(new IOException("Input Pattern " + p + " matches 0 files"));
}
else
{
foreach (FileStatus globStat in matches)
{
if (globStat.IsDirectory())
{
RemoteIterator <LocatedFileStatus> iter = fs.ListLocatedStatus(globStat.GetPath());
while (iter.HasNext())
{
LocatedFileStatus stat = iter.Next();
if (inputFilter.Accept(stat.GetPath()))
{
if (recursive && stat.IsDirectory())
{
AddInputPathRecursively(result, fs, stat.GetPath(), inputFilter);
}
else
{
result.AddItem(stat);
}
}
}
}
else
{
result.AddItem(globStat);
}
}
}
}
}
if (!errors.IsEmpty())
{
throw new InvalidInputException(errors);
}
return(result);
}
public void ShouldNotAcceptModificationsWithNullFilename()
{
Modification modification = new Modification();
modification.FolderName = "c:\\";
PathFilter filter = new PathFilter();
filter.Pattern = "c:\\*.*";
Assert.IsFalse(filter.Accept(modification));
}
public void RunTests()
{
int i = 0;
foreach (Modification m in modifications)
{
Assert.IsTrue(filter.Accept(m) == expectedResults[i],
DescribeExpectedResult(m, expectedResults[i]));
i++;
}
}
public void DeeplyNestedFilters()
{
Modification[] modlist = new Modification[]
{
ModificationMother.CreateModification("x.cs", "/working/sources"),
ModificationMother.CreateModification("Entries", "/working/sources/CVS"),
ModificationMother.CreateModification("x.build", "/working/build"),
ModificationMother.CreateModification("x.dll", "/working/build/target/sources")
};
PathFilter filter = new PathFilter();
filter.Pattern = "**/sources/**/*.*";
filter.Accept(modlist[0]);
}
/// <summary>Add files in the input path recursively into the results.</summary>
/// <param name="result">The List to store all files.</param>
/// <param name="fs">The FileSystem.</param>
/// <param name="path">The input path.</param>
/// <param name="inputFilter">The input filter that can be used to filter files/dirs.
/// </param>
/// <exception cref="System.IO.IOException"/>
protected internal virtual void AddInputPathRecursively(IList <FileStatus> result,
FileSystem fs, Path path, PathFilter inputFilter)
{
RemoteIterator <LocatedFileStatus> iter = fs.ListLocatedStatus(path);
while (iter.HasNext())
{
LocatedFileStatus stat = iter.Next();
if (inputFilter.Accept(stat.GetPath()))
{
if (stat.IsDirectory())
{
AddInputPathRecursively(result, fs, stat.GetPath(), inputFilter);
}
else
{
result.AddItem(stat);
}
}
}
}
/// <exception cref="System.IO.IOException"/>
private static IList <FileStatus> ListFilteredStatus(FileContext fc, Path root, PathFilter
filter)
{
IList <FileStatus> fsList = RemoteIterToList(fc.ListStatus(root));
if (filter == null)
{
return(fsList);
}
else
{
IList <FileStatus> filteredList = new List <FileStatus>();
foreach (FileStatus fs in fsList)
{
if (filter.Accept(fs.GetPath()))
{
filteredList.AddItem(fs);
}
}
return(filteredList);
}
}
public virtual bool Accept(Path path)
{
return(LogFilter.Accept(path));
}
/// <exception cref="System.IO.IOException"/>
public virtual FileStatus[] Glob()
{
// First we get the scheme and authority of the pattern that was passed
// in.
string scheme = SchemeFromPath(pathPattern);
string authority = AuthorityFromPath(pathPattern);
// Next we strip off everything except the pathname itself, and expand all
// globs. Expansion is a process which turns "grouping" clauses,
// expressed as brackets, into separate path patterns.
string pathPatternString = pathPattern.ToUri().GetPath();
IList <string> flattenedPatterns = GlobExpander.Expand(pathPatternString);
// Now loop over all flattened patterns. In every case, we'll be trying to
// match them to entries in the filesystem.
AList <FileStatus> results = new AList <FileStatus>(flattenedPatterns.Count);
bool sawWildcard = false;
foreach (string flatPattern in flattenedPatterns)
{
// Get the absolute path for this flattened pattern. We couldn't do
// this prior to flattening because of patterns like {/,a}, where which
// path you go down influences how the path must be made absolute.
Path absPattern = FixRelativePart(new Path(flatPattern.IsEmpty() ? Path.CurDir :
flatPattern));
// Now we break the flattened, absolute pattern into path components.
// For example, /a/*/c would be broken into the list [a, *, c]
IList <string> components = GetPathComponents(absPattern.ToUri().GetPath());
// Starting out at the root of the filesystem, we try to match
// filesystem entries against pattern components.
AList <FileStatus> candidates = new AList <FileStatus>(1);
// To get the "real" FileStatus of root, we'd have to do an expensive
// RPC to the NameNode. So we create a placeholder FileStatus which has
// the correct path, but defaults for the rest of the information.
// Later, if it turns out we actually want the FileStatus of root, we'll
// replace the placeholder with a real FileStatus obtained from the
// NameNode.
FileStatus rootPlaceholder;
if (Path.Windows && !components.IsEmpty() && Path.IsWindowsAbsolutePath(absPattern
.ToUri().GetPath(), true))
{
// On Windows the path could begin with a drive letter, e.g. /E:/foo.
// We will skip matching the drive letter and start from listing the
// root of the filesystem on that drive.
string driveLetter = components.Remove(0);
rootPlaceholder = new FileStatus(0, true, 0, 0, 0, new Path(scheme, authority, Path
.Separator + driveLetter + Path.Separator));
}
else
{
rootPlaceholder = new FileStatus(0, true, 0, 0, 0, new Path(scheme, authority, Path
.Separator));
}
candidates.AddItem(rootPlaceholder);
for (int componentIdx = 0; componentIdx < components.Count; componentIdx++)
{
AList <FileStatus> newCandidates = new AList <FileStatus>(candidates.Count);
GlobFilter globFilter = new GlobFilter(components[componentIdx]);
string component = UnescapePathComponent(components[componentIdx]);
if (globFilter.HasPattern())
{
sawWildcard = true;
}
if (candidates.IsEmpty() && sawWildcard)
{
// Optimization: if there are no more candidates left, stop examining
// the path components. We can only do this if we've already seen
// a wildcard component-- otherwise, we still need to visit all path
// components in case one of them is a wildcard.
break;
}
if ((componentIdx < components.Count - 1) && (!globFilter.HasPattern()))
{
// Optimization: if this is not the terminal path component, and we
// are not matching against a glob, assume that it exists. If it
// doesn't exist, we'll find out later when resolving a later glob
// or the terminal path component.
foreach (FileStatus candidate in candidates)
{
candidate.SetPath(new Path(candidate.GetPath(), component));
}
continue;
}
foreach (FileStatus candidate_1 in candidates)
{
if (globFilter.HasPattern())
{
FileStatus[] children = ListStatus(candidate_1.GetPath());
if (children.Length == 1)
{
// If we get back only one result, this could be either a listing
// of a directory with one entry, or it could reflect the fact
// that what we listed resolved to a file.
//
// Unfortunately, we can't just compare the returned paths to
// figure this out. Consider the case where you have /a/b, where
// b is a symlink to "..". In that case, listing /a/b will give
// back "/a/b" again. If we just went by returned pathname, we'd
// incorrectly conclude that /a/b was a file and should not match
// /a/*/*. So we use getFileStatus of the path we just listed to
// disambiguate.
if (!GetFileStatus(candidate_1.GetPath()).IsDirectory())
{
continue;
}
}
foreach (FileStatus child in children)
{
if (componentIdx < components.Count - 1)
{
// Don't try to recurse into non-directories. See HADOOP-10957.
if (!child.IsDirectory())
{
continue;
}
}
// Set the child path based on the parent path.
child.SetPath(new Path(candidate_1.GetPath(), child.GetPath().GetName()));
if (globFilter.Accept(child.GetPath()))
{
newCandidates.AddItem(child);
}
}
}
else
{
// When dealing with non-glob components, use getFileStatus
// instead of listStatus. This is an optimization, but it also
// is necessary for correctness in HDFS, since there are some
// special HDFS directories like .reserved and .snapshot that are
// not visible to listStatus, but which do exist. (See HADOOP-9877)
FileStatus childStatus = GetFileStatus(new Path(candidate_1.GetPath(), component)
);
if (childStatus != null)
{
newCandidates.AddItem(childStatus);
}
}
}
candidates = newCandidates;
}
foreach (FileStatus status in candidates)
{
// Use object equality to see if this status is the root placeholder.
// See the explanation for rootPlaceholder above for more information.
if (status == rootPlaceholder)
{
status = GetFileStatus(rootPlaceholder.GetPath());
if (status == null)
{
continue;
}
}
// HADOOP-3497 semantics: the user-defined filter is applied at the
// end, once the full path is built up.
if (filter.Accept(status.GetPath()))
{
results.AddItem(status);
}
}
}
/*
* When the input pattern "looks" like just a simple filename, and we
* can't find it, we return null rather than an empty array.
* This is a special case which the shell relies on.
*
* To be more precise: if there were no results, AND there were no
* groupings (aka brackets), and no wildcards in the input (aka stars),
* we return null.
*/
if ((!sawWildcard) && results.IsEmpty() && (flattenedPatterns.Count <= 1))
{
return(null);
}
return(Collections.ToArray(results, new FileStatus[0]));
}
public virtual bool Accept(Path path)
{
return(pattern.Matches(path.GetName()) && userFilter.Accept(path));
}