/// <summary>
/// Adds the output files of the pip to the set
/// </summary>
protected static void AddOutputs(IPipFilterContext context, PipId pipId, HashSet <FileOrDirectoryArtifact> outputs)
{
if (MayHaveOutputs(context, pipId))
{
AddOutputs(context.HydratePip(pipId), outputs);
}
}
/// <inheritdoc/>
public override IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(IPipFilterContext context, bool negate = false, IList <PipId> constrainingPips = null)
{
var matchingModulePipIds = ParallelProcessAllOutputs <PipId>(
context,
(pipId, localPips) =>
{
if (context.GetPipType(pipId) == PipType.Module)
{
var modulePip = (ModulePip)context.HydratePip(pipId);
if (m_modules.Contains(modulePip.Identity))
{
localPips.Add(pipId);
}
}
});
var dependenciesWithOutputs = GetDependenciesWithOutputsForModulePips(context, matchingModulePipIds);
if (constrainingPips != null)
{
dependenciesWithOutputs.IntersectWith(constrainingPips);
}
return(ParallelProcessAllOutputs <FileOrDirectoryArtifact>(
context,
(pipId, localOutputs) => AddOutputs(context, pipId, localOutputs),
dependenciesWithOutputs.ToArray()));
}
protected static ReadOnlyHashSet <T> ParallelProcessAllOutputs <T>(
IPipFilterContext context,
Action <PipId, HashSet <T> > action,
IList <PipId> pips = null)
{
var outputs = new ReadOnlyHashSet <T>();
pips = pips ?? context.AllPips;
// Note: pips better be an IList<...> in order to get good Parallel.ForEach performance
Parallel.ForEach(
pips,
() => new HashSet <T>(),
(pipId, loopState, index, localOutputs) =>
{
action(pipId, localOutputs);
return(localOutputs);
},
localOutputs =>
{
if (localOutputs.Count > 0)
{
lock (outputs)
{
// Even though the 'outputs' variable is of type ReadOnlyHashSet
// we still can modify it.
outputs.UnionWith(localOutputs);
}
}
});
return(outputs);
}
/// <inheritdoc/>
public override IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(IPipFilterContext context, bool negate = false, IList <PipId> constrainingPips = null)
{
return(ParallelProcessAllOutputs <FileOrDirectoryArtifact>(
context,
(pipId, localOutputs) => ForEachOutput(
localOutputs,
context,
pipId,
(localOutputs2, output) =>
{
if (output.IsFile)
{
if (PathMatches(output.Path, context.PathTable) ^ negate)
{
localOutputs2.Add(output);
}
}
else
{
// Directory can be non-output directory. See ForEachOutput.
if (output.DirectoryArtifact.IsOutputDirectory())
{
if (DirectoryPathMatches(output.Path, false, context.PathTable) ^ negate)
{
localOutputs2.Add(output);
}
}
}
}),
constrainingPips));
}
/// <inheritdoc/>
public override IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(
IPipFilterContext context,
bool negate = false,
IList <PipId> constrainingPips = null)
{
var matchingValuePipIds = ParallelProcessAllOutputs <PipId>(
context,
(pipId, localPips) =>
{
if (context.GetPipType(pipId) == PipType.Value)
{
ValuePip valuePip = (ValuePip)context.HydratePip(pipId);
// TODO: Consider not allowing matching of non-public values.
if (valuePip.Symbol == m_value ^ negate)
{
localPips.Add(pipId);
}
}
});
HashSet <PipId> dependenciesWithOutputs =
m_valueTransitive
? GetDependenciesWithOutputsBehindValueAndSealDirectoryPips(context, matchingValuePipIds)
: GetDependenciesWithOutputsForValuePips(context, matchingValuePipIds);
return(ParallelProcessAllOutputs <FileOrDirectoryArtifact>(
context,
(pipId, localOutputs) => AddOutputs(context, pipId, localOutputs),
dependenciesWithOutputs.ToArray()));
}
/// <inheritdoc/>
public override IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(IPipFilterContext context, bool negate = false, IList <PipId> constrainingPips = null)
{
var outputs = Inner.FilterOutputs(context, negate: false);
HashSet <PipId> innerPipProducers = new HashSet <PipId>();
foreach (var innerOutput in outputs)
{
innerPipProducers.Add(context.GetProducer(innerOutput));
}
var producersAndNeighbors = GetClosureWithOutputs(context, innerPipProducers, GetNeighborPips, ClosureMode);
if (IncludesInnerMatches)
{
// Add producers
producersAndNeighbors.UnionWith(innerPipProducers);
}
else
{
// Exclude inner matchs
producersAndNeighbors.ExceptWith(innerPipProducers);
}
// When not negated the set of pips to process is
// the intersection of the constraining pips and the producers and
// neighbors set.
if (!negate)
{
if (constrainingPips == null)
{
// No constraining pips, so only consider the producers and neighbors
constrainingPips = producersAndNeighbors.ToList();
}
else
{
// Has constraining pips, so intersect with the producers and neighbors
producersAndNeighbors.IntersectWith(constrainingPips);
constrainingPips = producersAndNeighbors.ToList();
}
}
return(ParallelProcessAllOutputs <FileOrDirectoryArtifact>(
context,
action: (pipId, localOutputs) =>
{
if (producersAndNeighbors.Contains(pipId) ^ negate)
{
ForEachOutput(
localOutputs,
context,
pipId,
(localOutputs2, output) => localOutputs2.Add(output));
}
},
pips: constrainingPips));
}
/// <summary>
/// Gets the pips that can have outputs in the transitive closure of a given set of value and seal directory pips
/// </summary>
protected static void AddTransitiveSpecDependencies(IPipFilterContext context, HashSet <PipId> specFileIds)
{
HashSet <PipId> closure = new HashSet <PipId>();
var stack = new Stack <PipId>();
foreach (var rootPipId in specFileIds)
{
closure.Add(rootPipId);
stack.Push(rootPipId);
}
while (stack.Count > 0)
{
var pipId = stack.Pop();
foreach (PipId dependency in context.GetDependencies(pipId))
{
if (closure.Add(dependency))
{
stack.Push(dependency);
}
}
var pipType = context.GetPipType(pipId);
if (pipType == PipType.Value)
{
// For value pips, get the spec corresponding spec file pip
foreach (var dependent in context.GetDependents(pipId))
{
if (context.GetPipType(dependent) == PipType.SpecFile)
{
if (specFileIds.Add(dependent))
{
stack.Push(dependent);
}
break;
}
}
}
else if (!pipType.IsMetaPip())
{
// Get the value pip for the pip and push to visit its dependencies and the corresponding spec file
foreach (var dependent in context.GetDependents(pipId))
{
if (context.GetPipType(dependent) == PipType.Value)
{
if (closure.Add(dependent))
{
stack.Push(dependent);
}
break;
}
}
}
}
}
private bool MatchesDependencies(IPipFilterContext context, Process proc)
{
foreach (var item in proc.Dependencies)
{
if (PathMatches(item.Path, context.PathTable))
{
return(true);
}
}
return(false);
}
/// <summary>
/// Determines whether a pip could have outputs
/// </summary>
protected static bool MayHaveOutputs(IPipFilterContext context, PipId pipId)
{
switch (context.GetPipType(pipId))
{
case PipType.Process:
case PipType.CopyFile:
case PipType.SealDirectory:
case PipType.WriteFile:
case PipType.Ipc:
return(true);
}
return(false);
}
/// <inheritdoc/>
public override IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(
IPipFilterContext context,
bool negate = false,
IList <PipId> constrainingPips = null)
{
return(ParallelProcessAllOutputs <FileOrDirectoryArtifact>(
context,
(pipId, localOutputs) =>
{
if ((context.GetSemiStableHash(pipId) == m_semiStableHash) ^ negate)
{
AddOutputs(context, pipId, localOutputs);
}
},
constrainingPips));
}
/// <inheritdoc />
public override IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(IPipFilterContext context, bool negate = false, IList <PipId> constrainingPips = null)
{
return(ParallelProcessAllOutputs <FileOrDirectoryArtifact>(
context,
(pipId, localOutputs) =>
{
if (MayHaveOutputs(context, pipId))
{
Pip pip = context.HydratePip(pipId);
if (Matches(pip) ^ negate)
{
AddOutputs(pip, localOutputs);
}
}
},
constrainingPips));
}
/// <summary>
/// Gets the pips that can have outputs which are direct dependencies of values of the given spec file pips.
/// </summary>
protected static HashSet <PipId> GetDependenciesWithOutputsForSpecFilePips(IPipFilterContext context, IEnumerable <PipId> specFilePipIds)
{
HashSet <PipId> valueDependencies = new HashSet <PipId>();
foreach (var specFilePipId in specFilePipIds)
{
foreach (var dependency in context.GetDependencies(specFilePipId))
{
if (context.GetPipType(dependency) == PipType.Value)
{
valueDependencies.Add(dependency);
}
}
}
return(GetDependenciesWithOutputsForValuePips(context, valueDependencies));
}
/// <summary>
/// Gets the pips that can have outputs which are direct dependencies of the given value pips.
/// </summary>
protected static HashSet <PipId> GetDependenciesWithOutputsForValuePips(IPipFilterContext context, IEnumerable <PipId> valuePipIds)
{
HashSet <PipId> dependenciesWithOutputs = new HashSet <PipId>();
foreach (var valuePipId in valuePipIds)
{
foreach (var dependency in context.GetDependencies(valuePipId))
{
if (MayHaveOutputs(context, dependency))
{
dependenciesWithOutputs.Add(dependency);
}
}
}
return(dependenciesWithOutputs);
}
/// <summary>
/// Gets the transitive dependent pips that can have outputs
/// </summary>
protected static HashSet <PipId> GetClosureWithOutputs(
IPipFilterContext context,
HashSet <PipId> pipIds,
Func <IPipFilterContext, PipId, IEnumerable <PipId> > getPips,
ClosureMode closureMode)
{
HashSet <PipId> closure = new HashSet <PipId>();
if (closureMode == ClosureMode.TransitiveIncludingSelf)
{
var stack = new Stack <PipId>();
foreach (var rootPipId in pipIds)
{
stack.Push(rootPipId);
}
while (stack.Count > 0)
{
var pipId = stack.Pop();
foreach (PipId neighbor in getPips(context, pipId))
{
if (MayHaveOutputs(context, neighbor) && closure.Add(neighbor))
{
stack.Push(neighbor);
}
}
}
}
else
{
Contract.Assert(closureMode == ClosureMode.DirectExcludingSelf);
foreach (var pipId in pipIds)
{
foreach (PipId neighbor in getPips(context, pipId))
{
if (MayHaveOutputs(context, neighbor))
{
closure.Add(neighbor);
}
}
}
}
return(closure);
}
/// <summary>
/// Returns the set of output files or directories that match the filter.
/// </summary>
/// <remarks>
/// NOTE: ALL FILTER OPERATION MUST BE IDEMPOTENT BECAUSE THE RESULTS ARE CACHED.
/// </remarks>
public IReadOnlySet <FileOrDirectoryArtifact> FilterOutputs(
IPipFilterContext context,
bool negate = false,
IList <PipId> constrainingPips = null)
{
if (negate || constrainingPips != null)
{
return(FilterOutputsCore(context, negate, constrainingPips));
}
if (context.TryGetCachedOutputs(this, out IReadOnlySet <FileOrDirectoryArtifact> result))
{
return(result);
}
result = FilterOutputsCore(context);
context.CacheOutputs(this, result);
return(result);
}
/// <inheritdoc/>
public override IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(IPipFilterContext context, bool negate = false, IList <PipId> constrainingPips = null)
{
var matchingSpecFilePipIds = ParallelProcessAllOutputs <PipId>(
context,
(pipId, localPips) =>
{
if (context.GetPipType(pipId) == PipType.SpecFile)
{
SpecFilePip specFilePip = (SpecFilePip)context.HydratePip(pipId);
if (PathMatches(specFilePip.SpecFile.Path, context.PathTable) ^ negate)
{
localPips.Add(pipId);
}
}
});
if (m_specDependencies)
{
AddTransitiveSpecDependencies(context, matchingSpecFilePipIds);
}
HashSet <PipId> dependenciesWithOutputs =
m_valueTransitive
? GetDependenciesWithOutputsBehindValueAndSealDirectoryPips(context, matchingSpecFilePipIds)
: GetDependenciesWithOutputsForSpecFilePips(context, matchingSpecFilePipIds);
if (constrainingPips != null)
{
dependenciesWithOutputs.IntersectWith(constrainingPips);
}
return(ParallelProcessAllOutputs <FileOrDirectoryArtifact>(
context,
(pipId, localOutputs) => AddOutputs(context, pipId, localOutputs),
dependenciesWithOutputs.ToArray()));
}
/// <inheritdoc/>
public override IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(IPipFilterContext context, bool negate = false, IList <PipId> constrainingPips = null)
{
// First we collect all matching seal directories
HashSet <DirectoryArtifact> directories = ParallelProcessAllOutputs <DirectoryArtifact>(
context,
(pipId, localDirectories) =>
{
if (context.GetPipType(pipId) == PipType.SealDirectory)
{
SealDirectory sd = (SealDirectory)context.HydratePip(pipId);
foreach (var item in sd.Contents)
{
if (PathMatches(item.Path, context.PathTable))
{
localDirectories.Add(sd.Directory);
break;
}
}
switch (sd.Kind)
{
case SealDirectoryKind.SourceAllDirectories:
case SealDirectoryKind.Opaque:
case SealDirectoryKind.SharedOpaque:
if (DirectoryPathMatches(sd.DirectoryRoot, false, context.PathTable))
{
localDirectories.Add(sd.Directory);
}
break;
case SealDirectoryKind.SourceTopDirectoryOnly:
if (DirectoryPathMatches(sd.DirectoryRoot, true, context.PathTable))
{
localDirectories.Add(sd.Directory);
}
break;
}
}
});
// Now look at all pips, checking if their input match one of the files or matching DirectoryArtifacts
return(ParallelProcessAllOutputs <FileOrDirectoryArtifact>(
context,
(pipId, localOutputs) =>
{
switch (context.GetPipType(pipId))
{
case PipType.CopyFile:
CopyFile cf = (CopyFile)context.HydratePip(pipId);
if (PathMatches(cf.Source, context.PathTable) ^ negate)
{
localOutputs.Add(FileOrDirectoryArtifact.Create(cf.Destination));
}
break;
case PipType.Process:
Process proc = (Process)context.HydratePip(pipId);
bool processMatches =
PathMatches(proc.Executable, context.PathTable) ||
MatchesDependencies(context, proc) ||
MatchesDirectoryDependencies(proc, directories);
if (processMatches ^ negate)
{
// TODO: If only directory dependencies matched, only include outputs from those directories
foreach (var output in proc.FileOutputs)
{
localOutputs.Add(FileOrDirectoryArtifact.Create(output.ToFileArtifact()));
}
foreach (var output in proc.DirectoryOutputs)
{
localOutputs.Add(FileOrDirectoryArtifact.Create(output));
}
}
break;
}
},
constrainingPips));
}
/// <inheritdoc/>
protected override IEnumerable <PipId> GetNeighborPips(IPipFilterContext context, PipId pip)
{
return(context.GetDependents(pip).Where(p => context.GetPipType(p) == BuildXL.Pips.Operations.PipType.CopyFile));
}
/// <inheritdoc/>
public override IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(IPipFilterContext context, bool negate = false, IList <PipId> constrainingPips = null)
{
Contract.Assert(false, "The empty filter should never be executed. It will always match so executing it is just wasted work.");
throw new InvalidOperationException("The empty filter should never be executed. It will always match so executing it is just wasted work.");
}
/// <inheritdoc/>
public override IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(IPipFilterContext context, bool negate = false, IList <PipId> constrainingPips = null)
{
return(Inner.FilterOutputs(context, !negate, constrainingPips));
}
/// <inheritdoc/>
protected override IEnumerable <PipId> GetNeighborPips(IPipFilterContext context, PipId pip)
{
return(context.GetDependents(pip));
}
/// <summary>
/// To be implemented by specific filter types. Returns the set of output files or directories that match the filter.
/// </summary>
/// <returns>Collection of output files that match the filter</returns>
public abstract IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(
IPipFilterContext context,
bool negate = false,
IList <PipId> constrainingPips = null);
/// <summary> /// Gets the neighbor (dependents or dependencies) pips of the current pip /// </summary> protected abstract IEnumerable <PipId> GetNeighborPips(IPipFilterContext context, PipId pip);
/// <summary>
/// Hydrates the pip and calls an action for each of the pip's outputs
/// (<see cref="ForEachOutput{TState}(TState,IPipFilterContext,PipId,Action{TState,FileOrDirectoryArtifact})"/>.
/// </summary>
protected static void ForEachOutput <TState>(TState state, IPipFilterContext context, PipId pipId, Action <TState, FileOrDirectoryArtifact> outputAction)
{
ForEachOutput(state, context.HydratePip(pipId), outputAction);
}
/// <summary>
/// Gets the pips that can have outputs in the transitive closure of a given set of value and seal directory pips
/// </summary>
protected static HashSet <PipId> GetDependenciesWithOutputsBehindValueAndSealDirectoryPips(IPipFilterContext context, IReadOnlySet <PipId> rootPipIds)
{
HashSet <PipId> closure = new HashSet <PipId>();
HashSet <PipId> dependenciesWithOutputs = new HashSet <PipId>();
var q = new Queue <PipId>();
foreach (var rootPipId in rootPipIds)
{
closure.Add(rootPipId);
q.Enqueue(rootPipId);
}
while (q.Count > 0)
{
var pipId = q.Dequeue();
foreach (PipId dependency in context.GetDependencies(pipId))
{
switch (context.GetPipType(dependency))
{
case PipType.Value:
case PipType.SealDirectory:
if (closure.Add(dependency))
{
q.Enqueue(dependency);
}
break;
case PipType.WriteFile:
case PipType.CopyFile:
case PipType.Process:
case PipType.Ipc:
dependenciesWithOutputs.Add(dependency);
break;
}
}
}
return(dependenciesWithOutputs);
}
