private static RecursiveStepResult GetFilesRecursiveStep
(
RecursionState recursionState,
string projectDirectory,
bool stripProjectDirectory,
GetFileSystemEntries getFileSystemEntries
)
{
RecursiveStepResult ret = new RecursiveStepResult();
/*
* Get the matching files.
*/
bool considerFiles = false;
// Only consider files if...
if (recursionState.RemainingWildcardDirectory.Length == 0)
{
// We've reached the end of the wildcard directory elements.
considerFiles = true;
}
else if (recursionState.RemainingWildcardDirectory.IndexOf(recursiveDirectoryMatch, StringComparison.Ordinal) == 0)
{
// or, we've reached a "**" so everything else is matched recursively.
considerFiles = true;
}
if (considerFiles)
{
string[] files = getFileSystemEntries(FileSystemEntity.Files, recursionState.BaseDirectory,
recursionState.SearchData.Filespec, projectDirectory, stripProjectDirectory);
bool needToProcessEachFile = recursionState.SearchData.Filespec == null || recursionState.SearchData.ExtensionLengthToEnforce != 0;
if (needToProcessEachFile)
{
List<string> listOfFiles = new List<string>();
foreach (string file in files)
{
if ((recursionState.SearchData.Filespec != null) ||
// if no file-spec provided, match the file to the regular expression
// PERF NOTE: Regex.IsMatch() is an expensive operation, so we avoid it whenever possible
recursionState.SearchData.RegexFileMatch.IsMatch(file))
{
if ((recursionState.SearchData.Filespec == null) ||
// if we used a file-spec with a "loosely" defined extension
(recursionState.SearchData.ExtensionLengthToEnforce == 0) ||
// discard all files that do not have extensions of the desired length
(Path.GetExtension(file).Length == recursionState.SearchData.ExtensionLengthToEnforce))
{
listOfFiles.Add(file);
}
}
}
ret.Files = listOfFiles.ToArray();
}
else
{
ret.Files = files;
}
}
/*
* Recurse into subdirectories.
*/
if (recursionState.SearchData.NeedsRecursion && recursionState.RemainingWildcardDirectory.Length > 0)
{
// Find the next directory piece.
string pattern = null;
if (!IsRecursiveDirectoryMatch(recursionState.RemainingWildcardDirectory))
{
int indexOfNextSlash = recursionState.RemainingWildcardDirectory.IndexOfAny(directorySeparatorCharacters);
ErrorUtilities.VerifyThrow(indexOfNextSlash != -1, "Slash should be guaranteed.");
pattern = recursionState.RemainingWildcardDirectory.Substring(0, indexOfNextSlash);
if (pattern == recursiveDirectoryMatch)
{
// If pattern turned into **, then there's no choice but to enumerate everything.
pattern = null;
recursionState.RemainingWildcardDirectory = recursiveDirectoryMatch;
}
else
{
// Peel off the leftmost directory piece. So for example, if remainingWildcardDirectory
// contains:
//
// ?emp\foo\**\bar
//
// then put '?emp' into pattern. Then put the remaining part,
//
// foo\**\bar
//
// back into remainingWildcardDirectory.
// This is a performance optimization. We don't want to enumerate everything if we
// don't have to.
recursionState.RemainingWildcardDirectory = recursionState.RemainingWildcardDirectory.Substring(indexOfNextSlash + 1);
}
}
ret.RemainingWildcardDirectory = recursionState.RemainingWildcardDirectory;
ret.Subdirs = getFileSystemEntries(FileSystemEntity.Directories, recursionState.BaseDirectory, pattern, null, false);
}
return ret;
}
static SearchAction GetFileSearchData(string projectDirectoryUnescaped, string filespecUnescaped,
GetFileSystemEntries getFileSystemEntries, DirectoryExists directoryExists, out bool stripProjectDirectory,
out RecursionState result)
{
stripProjectDirectory = false;
result = new RecursionState();
string fixedDirectoryPart;
string wildcardDirectoryPart;
string filenamePart;
string matchFileExpression;
bool needsRecursion;
bool isLegalFileSpec;
GetFileSpecInfo
(
filespecUnescaped,
out fixedDirectoryPart,
out wildcardDirectoryPart,
out filenamePart,
out matchFileExpression,
out needsRecursion,
out isLegalFileSpec,
getFileSystemEntries
);
/*
* If the filespec is invalid, then just return now.
*/
if (!isLegalFileSpec)
{
return SearchAction.ReturnFileSpec;
}
// The projectDirectory is not null only if we are running the evaluation from
// inside the engine (i.e. not from a task)
if (projectDirectoryUnescaped != null)
{
if (fixedDirectoryPart != null)
{
string oldFixedDirectoryPart = fixedDirectoryPart;
try
{
fixedDirectoryPart = Path.Combine(projectDirectoryUnescaped, fixedDirectoryPart);
}
catch (ArgumentException)
{
return SearchAction.ReturnEmptyList;
}
stripProjectDirectory = !String.Equals(fixedDirectoryPart, oldFixedDirectoryPart, StringComparison.OrdinalIgnoreCase);
}
else
{
fixedDirectoryPart = projectDirectoryUnescaped;
stripProjectDirectory = true;
}
}
/*
* If the fixed directory part doesn't exist, then this means no files should be
* returned.
*/
if (fixedDirectoryPart.Length > 0 && !directoryExists(fixedDirectoryPart))
{
return SearchAction.ReturnEmptyList;
}
// determine if we need to use the regular expression to match the files
// PERF NOTE: Constructing a Regex object is expensive, so we avoid it whenever possible
bool matchWithRegex =
// if we have a directory specification that uses wildcards, and
(wildcardDirectoryPart.Length > 0) &&
// the specification is not a simple "**"
!IsRecursiveDirectoryMatch(wildcardDirectoryPart);
// then we need to use the regular expression
// if we're not using the regular expression, get the file pattern extension
string extensionPart = matchWithRegex
? null
: Path.GetExtension(filenamePart);
// check if the file pattern would cause Windows to match more loosely on the extension
// NOTE: Windows matches loosely in two cases (in the absence of the * wildcard in the extension):
// 1) if the extension ends with the ? wildcard, it matches files with shorter extensions also e.g. "file.tx?" would
// match both "file.txt" and "file.tx"
// 2) if the extension is three characters, and the filename contains the * wildcard, it matches files with longer
// extensions that start with the same three characters e.g. "*.htm" would match both "file.htm" and "file.html"
bool needToEnforceExtensionLength =
(extensionPart != null) &&
(extensionPart.IndexOf('*') == -1)
&&
(extensionPart.EndsWith("?", StringComparison.Ordinal)
||
((extensionPart.Length == (3 + 1 /* +1 for the period */)) &&
(filenamePart.IndexOf('*') != -1)));
var searchData = new FilesSearchData(
// if using the regular expression, ignore the file pattern
(matchWithRegex ? null : filenamePart), (needToEnforceExtensionLength ? extensionPart.Length : 0),
// if using the file pattern, ignore the regular expression
(matchWithRegex ? new Regex(matchFileExpression, RegexOptions.IgnoreCase) : null),
needsRecursion);
result.SearchData = searchData;
result.BaseDirectory = fixedDirectoryPart;
result.RemainingWildcardDirectory = wildcardDirectoryPart;
return SearchAction.RunSearch;
}
/// <summary>
/// Get all files that match either the file-spec or the regular expression.
/// </summary>
/// <param name="listOfFiles">List of files that gets populated.</param>
/// <param name="recursionState">Information about the search</param>
/// <param name="projectDirectory"></param>
/// <param name="stripProjectDirectory"></param>
/// <param name="getFileSystemEntries">Delegate.</param>
/// <param name="searchesToExclude">Patterns to exclude from the results</param>
/// <param name="searchesToExcludeInSubdirs">exclude patterns that might activate farther down the directory tree. Keys assume paths are normalized with forward slashes and no trailing slashes</param>
private static void GetFilesRecursive
(
IList<string> listOfFiles,
RecursionState recursionState,
string projectDirectory,
bool stripProjectDirectory,
GetFileSystemEntries getFileSystemEntries,
IList<RecursionState> searchesToExclude,
Dictionary<string, List<RecursionState>> searchesToExcludeInSubdirs
)
{
ErrorUtilities.VerifyThrow((recursionState.SearchData.Filespec== null) || (recursionState.SearchData.RegexFileMatch == null),
"File-spec overrides the regular expression -- pass null for file-spec if you want to use the regular expression.");
ErrorUtilities.VerifyThrow((recursionState.SearchData.Filespec != null) || (recursionState.SearchData.RegexFileMatch != null),
"Need either a file-spec or a regular expression to match files.");
ErrorUtilities.VerifyThrow(recursionState.RemainingWildcardDirectory != null, "Expected non-null remaning wildcard directory.");
// Determine if any of searchesToExclude is necessarily a superset of the results that will be returned.
// This means all results will be excluded and we should bail out now.
if (searchesToExclude != null)
{
foreach (var searchToExclude in searchesToExclude)
{
// The BaseDirectory of all the exclude searches should be the same as the include one
Debug.Assert(FileUtilities.PathsEqual(searchToExclude.BaseDirectory, recursionState.BaseDirectory), "Expected exclude search base directory to match include search base directory");
// We can exclude all results in this folder if:
if (
// We are matching files based on a filespec and not a regular expression
searchToExclude.SearchData.Filespec != null &&
// The wildcard path portion of the excluded search matches the include search
FileUtilities.PathsEqual(searchToExclude.RemainingWildcardDirectory, recursionState.RemainingWildcardDirectory) &&
// The exclude search will match ALL filenames OR
(searchToExclude.SearchData.Filespec == "*" || searchToExclude.SearchData.Filespec == "*.*" ||
// The exclude search filename pattern matches the include search's pattern
(searchToExclude.SearchData.Filespec == recursionState.SearchData.Filespec &&
searchToExclude.SearchData.ExtensionLengthToEnforce == recursionState.SearchData.ExtensionLengthToEnforce)))
{
// We won't get any results from this search that we would end up keeping
return;
}
}
}
RecursiveStepResult nextStep = GetFilesRecursiveStep(
recursionState,
projectDirectory,
stripProjectDirectory,
getFileSystemEntries);
RecursiveStepResult[] excludeNextSteps = null;
if (searchesToExclude != null)
{
excludeNextSteps = new RecursiveStepResult[searchesToExclude.Count];
for (int i = 0; i < searchesToExclude.Count; i++)
{
excludeNextSteps[i] = GetFilesRecursiveStep(
searchesToExclude[i],
projectDirectory,
stripProjectDirectory,
getFileSystemEntries);
}
}
if (nextStep.Files != null)
{
HashSet<string> filesToExclude = null;
if (excludeNextSteps != null)
{
filesToExclude = new HashSet<string>();
foreach (var excludeStep in excludeNextSteps)
{
foreach (var file in excludeStep.Files)
{
filesToExclude.Add(file);
}
}
}
foreach (var file in nextStep.Files)
{
if (filesToExclude == null || !filesToExclude.Contains(file))
{
listOfFiles.Add(file);
}
}
}
if (nextStep.Subdirs != null)
{
foreach (string subdir in nextStep.Subdirs)
{
// RecursionState is a struct so this copies it
var newRecursionState = recursionState;
newRecursionState.BaseDirectory = subdir;
newRecursionState.RemainingWildcardDirectory = nextStep.RemainingWildcardDirectory;
List<RecursionState> newSearchesToExclude = null;
if (excludeNextSteps != null)
{
newSearchesToExclude = new List<RecursionState>();
for (int i = 0; i < excludeNextSteps.Length; i++)
{
if (excludeNextSteps[i].Subdirs != null &&
excludeNextSteps[i].Subdirs.Any(excludedDir => FileUtilities.PathsEqual(excludedDir, subdir)))
{
RecursionState thisExcludeStep = searchesToExclude[i];
thisExcludeStep.BaseDirectory = subdir;
thisExcludeStep.RemainingWildcardDirectory = excludeNextSteps[i].RemainingWildcardDirectory;
newSearchesToExclude.Add(thisExcludeStep);
}
}
}
if (searchesToExcludeInSubdirs != null)
{
List<RecursionState> searchesForSubdir;
// The normalization fixes https://github.com/Microsoft/msbuild/issues/917
// and is a partial fix for https://github.com/Microsoft/msbuild/issues/724
if (searchesToExcludeInSubdirs.TryGetValue(subdir.NormalizeForPathComparison(), out searchesForSubdir))
{
// We've found the base directory that these exclusions apply to. So now add them as normal searches
if (newSearchesToExclude == null)
{
newSearchesToExclude = new List<RecursionState>();
}
newSearchesToExclude.AddRange(searchesForSubdir);
}
}
// We never want to strip the project directory from the leaves, because the current
// process directory maybe different
GetFilesRecursive(
listOfFiles,
newRecursionState,
projectDirectory,
stripProjectDirectory,
getFileSystemEntries,
newSearchesToExclude,
searchesToExcludeInSubdirs);
}
}
}
public void RecalculateExpanded(out List <string> warnings)
{
// TODO: A performance improvement could be made by short-circuiting if the non-expanded values are not dirty. It's unclear whether it would make a significant difference.
// Take a lock to prevent the variables from changing while expansion is being processed.
lock (_setLock)
{
const int MaxDepth = 50;
// TODO: Validate max size? No limit on *nix. Max of 32k per env var on Windows https://msdn.microsoft.com/en-us/library/windows/desktop/ms682653%28v=vs.85%29.aspx
_trace.Entering();
warnings = new List <string>();
// Create a new expanded instance.
var expanded = new ConcurrentDictionary <string, Variable>(_nonexpanded, StringComparer.OrdinalIgnoreCase);
// Process each variable in the dictionary.
foreach (string name in _nonexpanded.Keys)
{
bool secret = _nonexpanded[name].Secret;
bool readOnly = _nonexpanded[name].ReadOnly;
_trace.Verbose($"Processing expansion for variable: '{name}'");
// This algorithm handles recursive replacement using a stack.
// 1) Max depth is enforced by leveraging the stack count.
// 2) Cyclical references are detected by walking the stack.
// 3) Additional call frames are avoided.
bool exceedsMaxDepth = false;
bool hasCycle = false;
var stack = new Stack <RecursionState>();
RecursionState state = new RecursionState(name: name, value: _nonexpanded[name].Value ?? string.Empty);
// The outer while loop is used to manage popping items from the stack (of state objects).
while (true)
{
// The inner while loop is used to manage replacement within the current state object.
// Find the next macro within the current value.
while (state.StartIndex < state.Value.Length &&
(state.PrefixIndex = state.Value.IndexOf(Constants.Variables.MacroPrefix, state.StartIndex, StringComparison.Ordinal)) >= 0 &&
(state.SuffixIndex = state.Value.IndexOf(Constants.Variables.MacroSuffix, state.PrefixIndex + Constants.Variables.MacroPrefix.Length, StringComparison.Ordinal)) >= 0)
{
// A candidate was found.
string nestedName = state.Value.Substring(
startIndex: state.PrefixIndex + Constants.Variables.MacroPrefix.Length,
length: state.SuffixIndex - state.PrefixIndex - Constants.Variables.MacroPrefix.Length);
if (!secret)
{
_trace.Verbose($"Found macro candidate: '{nestedName}'");
}
Variable nestedVariable;
if (!string.IsNullOrEmpty(nestedName) &&
_nonexpanded.TryGetValue(nestedName, out nestedVariable))
{
// A matching variable was found.
// Check for max depth.
int currentDepth = stack.Count + 1; // Add 1 since the current state isn't on the stack.
if (currentDepth == MaxDepth)
{
// Warn and break out of the while loops.
_trace.Warning("Exceeds max depth.");
exceedsMaxDepth = true;
warnings.Add(StringUtil.Loc("Variable0ExceedsMaxDepth1", name, MaxDepth));
break;
}
// Check for a cyclical reference.
else if (string.Equals(state.Name, nestedName, StringComparison.OrdinalIgnoreCase) ||
stack.Any(x => string.Equals(x.Name, nestedName, StringComparison.OrdinalIgnoreCase)))
{
// Warn and break out of the while loops.
_trace.Warning("Cyclical reference detected.");
hasCycle = true;
warnings.Add(StringUtil.Loc("Variable0ContainsCyclicalReference", name));
break;
}
else
{
// Push the current state and start a new state. There is no need to break out
// of the inner while loop. It will continue processing the new current state.
secret = secret || nestedVariable.Secret;
if (!secret)
{
_trace.Verbose($"Processing expansion for nested variable: '{nestedName}'");
}
stack.Push(state);
state = new RecursionState(name: nestedName, value: StringTranslator(nestedVariable.Value ?? string.Empty));
}
}
else
{
// A matching variable was not found.
if (!secret)
{
_trace.Verbose("Macro not found.");
}
state.StartIndex = state.PrefixIndex + 1;
}
} // End of inner while loop for processing the variable.
// No replacement is performed if something went wrong.
if (exceedsMaxDepth || hasCycle)
{
break;
}
// Check if finished processing the stack.
if (stack.Count == 0)
{
// Store the final value and break out of the outer while loop.
if (!string.Equals(state.Value, _nonexpanded[name].Value, StringComparison.Ordinal))
{
// Register the secret.
if (secret && !string.IsNullOrEmpty(state.Value))
{
_secretMasker.AddValue(state.Value);
}
// Set the expanded value.
expanded[state.Name] = new Variable(state.Name, state.Value, secret, readOnly);
_trace.Verbose($"Set '{state.Name}' = '{state.Value}'");
}
break;
}
// Adjust and pop the parent state.
if (!secret)
{
_trace.Verbose("Popping recursion state.");
}
RecursionState parent = stack.Pop();
parent.Value = string.Concat(
parent.Value.Substring(0, parent.PrefixIndex),
state.Value,
parent.Value.Substring(parent.SuffixIndex + Constants.Variables.MacroSuffix.Length));
parent.StartIndex = parent.PrefixIndex + (state.Value).Length;
state = parent;
if (!secret)
{
_trace.Verbose($"Intermediate state '{state.Name}': '{state.Value}'");
}
} // End of outer while loop for recursively processing the variable.
} // End of foreach loop over each key in the dictionary.
_expanded = expanded;
} // End of critical section.
}
