/// <summary>
/// Gets the outputs produced by a pip and calls an action
/// </summary>
public static bool ForEachOutput(Pip pip, Func <FileOrDirectoryArtifact, bool> outputAction, bool includeUncacheable)
{
bool result = true;
switch (pip.PipType)
{
case PipType.CopyFile:
CopyFile copyFile = (CopyFile)pip;
result = outputAction(FileOrDirectoryArtifact.Create(copyFile.Destination));
break;
case PipType.SealDirectory:
SealDirectory sealDirectory = (SealDirectory)pip;
result = outputAction(FileOrDirectoryArtifact.Create(sealDirectory.Directory));
break;
case PipType.Process:
Process process = (Process)pip;
foreach (var output in process.FileOutputs)
{
if (includeUncacheable || output.CanBeReferencedOrCached())
{
if (!outputAction(FileOrDirectoryArtifact.Create(output.ToFileArtifact())))
{
return(false);
}
}
}
foreach (var output in process.DirectoryOutputs)
{
if (!outputAction(FileOrDirectoryArtifact.Create(output)))
{
return(false);
}
}
break;
case PipType.WriteFile:
WriteFile writeFile = (WriteFile)pip;
result = outputAction(FileOrDirectoryArtifact.Create(writeFile.Destination));
break;
case PipType.Ipc:
IpcPip ipcPip = (IpcPip)pip;
result = outputAction(FileOrDirectoryArtifact.Create(ipcPip.OutputFile));
break;
}
return(result);
}
/// <summary>
/// Gets the outputs produced by a pip and calls an action
/// </summary>
protected static void ForEachOutput <TState>(TState state, Pip pip, Action <TState, FileOrDirectoryArtifact> outputAction)
{
switch (pip.PipType)
{
case PipType.CopyFile:
CopyFile copyFile = (CopyFile)pip;
outputAction(state, FileOrDirectoryArtifact.Create(copyFile.Destination));
break;
case PipType.Process:
Process process = (Process)pip;
foreach (var output in process.FileOutputs)
{
outputAction(state, FileOrDirectoryArtifact.Create(output.ToFileArtifact()));
}
foreach (var output in process.DirectoryOutputs)
{
outputAction(state, FileOrDirectoryArtifact.Create(output));
}
break;
case PipType.WriteFile:
WriteFile writeFile = (WriteFile)pip;
outputAction(state, FileOrDirectoryArtifact.Create(writeFile.Destination));
break;
case PipType.SealDirectory:
SealDirectory sealDirectory = (SealDirectory)pip;
outputAction(state, FileOrDirectoryArtifact.Create(sealDirectory.Directory));
break;
case PipType.Ipc:
IpcPip ipcPip = (IpcPip)pip;
outputAction(state, FileOrDirectoryArtifact.Create(ipcPip.OutputFile));
break;
}
}
public void AddRealPath(string fullPath, PathExistence existence)
{
if (existence == PathExistence.Nonexistent)
{
return;
}
var path = AbsolutePath.Create(m_pathTable, fullPath);
FileOrDirectoryArtifact member = FileOrDirectoryArtifact.Invalid;
while (path.IsValid)
{
if (existence == PathExistence.ExistsAsFile)
{
m_files.Add(path);
member = new FileArtifact(path);
}
else
{
var members = m_directories.GetOrAdd(path, p => new HashSet <FileOrDirectoryArtifact>());
if (member.IsValid)
{
if (!members.Add(member))
{
return;
}
}
member = DirectoryArtifact.CreateWithZeroPartialSealId(path);
}
existence = PathExistence.ExistsAsDirectory;
path = path.GetParent(m_pathTable);
}
}
/// <summary>
/// Matches the string representation of <see cref="FileOrDirectoryArtifact"/> used by the fingerprint store
/// when serializing to JSON.
/// </summary>
private string ArtifactToPrint(FileOrDirectoryArtifact artifact)
{
return(Expander.ExpandPath(Context.PathTable, artifact.Path).ToLowerInvariant().Replace(@"\", @"\\"));
}
/*** TO STRING HELPER FUNCTIONS ***/
/// <summary>
/// Converts FileOrDirectoryArtifact to string
/// </summary>
public string FileOrDirectoryToString(FileOrDirectoryArtifact fileArtifact)
{
return(fileArtifact.Path.ToString(PathTable));
}
/// <summary>
/// Creates a create symbolic link operation
/// </summary>
/// <remarks>
/// This method is for testing symlinks whose target either
/// (a) cannot be represented using FileArtifact/AbsolutePath (e.g., path contains an illegal char), or
/// (b) should not be represented using these structs (e.g., target's path is a relative path)
/// </remarks>
public static Operation CreateSymlink(FileOrDirectoryArtifact linkPath, string target, SymbolicLinkFlag symLinkFlag, bool doNotInfer = false)
{
return(new Operation(Type.CreateSymlink, linkPath: linkPath, additionalArgs: target, symLinkFlag: symLinkFlag, doNotInfer: doNotInfer));
}
/// <summary>
/// Creates a probe operation
/// </summary>
public static Operation Probe(FileOrDirectoryArtifact path, bool doNotInfer = false)
{
return(new Operation(Type.Probe, path, doNotInfer: doNotInfer));
}
/// <summary>
/// Creates a create directory operation (uses WinAPI)
/// </summary>
public static Operation CreateDir(FileOrDirectoryArtifact path, bool doNotInfer = false)
{
return(new Operation(Type.CreateDir, path, doNotInfer: doNotInfer));
}
private bool TryScheduleIpcPip(Context context, ObjectLiteral obj, bool allowUndefinedTargetService, bool isServiceFinalization, out FileArtifact outputFile, out PipId pipId)
{
// IpcClientInfo
IIpcMoniker moniker = Converter.ExtractRef <IIpcMoniker>(obj, m_ipcSendMoniker, allowUndefined: false);
int? numRetries = Converter.ExtractNumber(obj, m_ipcSendMaxConnectRetries, allowUndefined: true);
int? retryDelayMillis = Converter.ExtractNumber(obj, m_ipcSendConnectRetryDelayMillis, allowUndefined: true);
var clientConfig = new ClientConfig(numRetries, retryDelayMillis);
var ipcClientInfo = new IpcClientInfo(moniker.ToStringId(context.StringTable), clientConfig);
// target service pip
PipId?servicePipId = Converter.ExtractValue <PipId>(obj, m_ipcSendTargetServicePip, allowUndefined: allowUndefinedTargetService);
// arguments
PipData arguments;
ReadOnlyArray <FileArtifact> fileDependencies;
ReadOnlyArray <DirectoryArtifact> directoryDependencies;
using (var ipcProcessBuilder = ProcessBuilder.Create(context.PathTable, context.FrontEndContext.GetPipDataBuilder()))
{
// process arguments
ArrayLiteral argumentsArrayLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendMessageBody);
TransformerExecuteArgumentsProcessor.ProcessArguments(context, ipcProcessBuilder, argumentsArrayLiteral);
// input file dependencies
var dependenciesArray = Converter.ExtractArrayLiteral(obj, m_ipcSendDependencies, allowUndefined: true);
if (dependenciesArray != null)
{
for (int i = 0; i < dependenciesArray.Length; i++)
{
ProcessImplicitDependency(ipcProcessBuilder, dependenciesArray[i], convContext: new ConversionContext(pos: i, objectCtx: dependenciesArray));
}
}
arguments = ipcProcessBuilder.ArgumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
fileDependencies = ipcProcessBuilder.GetInputFilesSoFar();
directoryDependencies = ipcProcessBuilder.GetInputDirectoriesSoFar();
}
// output
AbsolutePath output = Converter.ExtractPath(obj, m_ipcSendOutputFile, allowUndefined: true);
if (!output.IsValid)
{
output = context.GetPipConstructionHelper().GetUniqueObjectDirectory(m_ipcObjectFolderName).Path.Combine(context.PathTable, m_ipcOutputFileName);
}
// tags
string[] tags = null;
var tagsArray = Converter.ExtractArrayLiteral(obj, m_executeTags, allowUndefined: true);
if (tagsArray != null && tagsArray.Count > 0)
{
tags = new string[tagsArray.Count];
for (int i = 0; i < tagsArray.Count; i++)
{
tags[i] = Converter.ExpectString(tagsArray[i], context: new ConversionContext(pos: i, objectCtx: tagsArray));
}
}
// skip materialization for files
FileOrDirectoryArtifact[] skipMaterializationArtifacts = CollectionUtilities.EmptyArray <FileOrDirectoryArtifact>();
ArrayLiteral skipMaterializationLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendLazilyMaterializedDependencies, allowUndefined: true);
if (skipMaterializationLiteral != null)
{
skipMaterializationArtifacts = new FileOrDirectoryArtifact[skipMaterializationLiteral.Length];
for (int i = 0; i < skipMaterializationLiteral.Length; i++)
{
Converter.ExpectFileOrStaticDirectory(
skipMaterializationLiteral[i],
out var fileArtifact,
out var staticDirectory,
context: new ConversionContext(pos: i, objectCtx: skipMaterializationLiteral));
Contract.Assert(fileArtifact.IsValid ^ staticDirectory != null);
skipMaterializationArtifacts[i] = fileArtifact.IsValid
? FileOrDirectoryArtifact.Create(fileArtifact)
: FileOrDirectoryArtifact.Create(staticDirectory.Root);
}
}
// must run on master
var mustRunOnMaster = Converter.ExtractOptionalBoolean(obj, m_ipcSendMustRunOnMaster) == true;
outputFile = FileArtifact.CreateOutputFile(output);
// create IPC pip and add it to the graph
bool result = context.GetPipConstructionHelper().TryAddIpc(
ipcClientInfo,
arguments,
outputFile,
servicePipDependencies: servicePipId != null ? ReadOnlyArray <PipId> .From(new[] { servicePipId.Value }) : ReadOnlyArray <PipId> .Empty,
fileDependencies: fileDependencies,
directoryDependencies: directoryDependencies,
skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .FromWithoutCopy(skipMaterializationArtifacts),
isServiceFinalization: isServiceFinalization,
mustRunOnMaster: mustRunOnMaster,
tags: tags,
out var ipcPip);
pipId = ipcPip.PipId;
return(result);
}
private IExpression Generate(FileOrDirectoryArtifact artifact)
{
return(Generate(artifact.Path, artifact.IsFile ? "f" : "d"));
}
/// <summary>
/// Creates a create directory operation (uses WinAPI)
/// </summary>
public static Operation CreateDir(FileOrDirectoryArtifact path, bool doNotInfer = false, string additionalArgs = null)
{
return(new Operation(Type.CreateDir, path, doNotInfer: doNotInfer, additionalArgs: additionalArgs));
}
public string ArtifactToString(FileOrDirectoryArtifact file, PathTable pathTable = null)
{
return(file.Path.ToString(pathTable ?? Context.PathTable));
}
/// <summary>
/// Gets the inputs consumed by a pip and calls an action
/// </summary>
public static bool ForEachInput(
Pip pip,
Func <FileOrDirectoryArtifact, bool> inputAction,
bool includeLazyInputs,
Func <FileOrDirectoryArtifact, bool> overrideLazyInputAction = null)
{
// NOTE: Lazy inputs must be processed AFTER regular inputs
// This behavior is required by FileContentManager.PopulateDepdencies
bool result = true;
switch (pip.PipType)
{
case PipType.CopyFile:
CopyFile copyFile = (CopyFile)pip;
result = inputAction(FileOrDirectoryArtifact.Create(copyFile.Source));
break;
case PipType.Process:
Process process = (Process)pip;
foreach (var input in process.Dependencies)
{
if (!inputAction(FileOrDirectoryArtifact.Create(input)))
{
return(false);
}
}
foreach (var input in process.DirectoryDependencies)
{
if (!inputAction(FileOrDirectoryArtifact.Create(input)))
{
return(false);
}
}
break;
case PipType.SealDirectory:
SealDirectory sealDirectory = (SealDirectory)pip;
foreach (var input in sealDirectory.Contents)
{
if (!inputAction(FileOrDirectoryArtifact.Create(input)))
{
return(false);
}
}
break;
case PipType.Ipc:
IpcPip ipcPip = (IpcPip)pip;
foreach (var input in ipcPip.FileDependencies)
{
if (!inputAction(FileOrDirectoryArtifact.Create(input)))
{
return(false);
}
}
foreach (var input in ipcPip.DirectoryDependencies)
{
if (!inputAction(FileOrDirectoryArtifact.Create(input)))
{
return(false);
}
}
if (includeLazyInputs)
{
overrideLazyInputAction = overrideLazyInputAction ?? inputAction;
foreach (var input in ipcPip.LazilyMaterializedDependencies)
{
if (!overrideLazyInputAction(input))
{
return(false);
}
}
}
break;
}
return(result);
}
/// <inheritdoc/>
public override IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(IPipFilterContext context, bool negate = false, IList <PipId> constrainingPips = null)
{
// First we collect all matching seal directories
HashSet <DirectoryArtifact> directories = ParallelProcessAllOutputs <DirectoryArtifact>(
context,
(pipId, localDirectories) =>
{
if (context.GetPipType(pipId) == PipType.SealDirectory)
{
SealDirectory sd = (SealDirectory)context.HydratePip(pipId);
foreach (var item in sd.Contents)
{
if (PathMatches(item.Path, context.PathTable))
{
localDirectories.Add(sd.Directory);
break;
}
}
switch (sd.Kind)
{
case SealDirectoryKind.SourceAllDirectories:
case SealDirectoryKind.Opaque:
case SealDirectoryKind.SharedOpaque:
if (DirectoryPathMatches(sd.DirectoryRoot, false, context.PathTable))
{
localDirectories.Add(sd.Directory);
}
break;
case SealDirectoryKind.SourceTopDirectoryOnly:
if (DirectoryPathMatches(sd.DirectoryRoot, true, context.PathTable))
{
localDirectories.Add(sd.Directory);
}
break;
}
}
});
// Now look at all pips, checking if their input match one of the files or matching DirectoryArtifacts
return(ParallelProcessAllOutputs <FileOrDirectoryArtifact>(
context,
(pipId, localOutputs) =>
{
switch (context.GetPipType(pipId))
{
case PipType.CopyFile:
CopyFile cf = (CopyFile)context.HydratePip(pipId);
if (PathMatches(cf.Source, context.PathTable) ^ negate)
{
localOutputs.Add(FileOrDirectoryArtifact.Create(cf.Destination));
}
break;
case PipType.Process:
Process proc = (Process)context.HydratePip(pipId);
bool processMatches =
PathMatches(proc.Executable, context.PathTable) ||
MatchesDependencies(context, proc) ||
MatchesDirectoryDependencies(proc, directories);
if (processMatches ^ negate)
{
// TODO: If only directory dependencies matched, only include outputs from those directories
foreach (var output in proc.FileOutputs)
{
localOutputs.Add(FileOrDirectoryArtifact.Create(output.ToFileArtifact()));
}
foreach (var output in proc.DirectoryOutputs)
{
localOutputs.Add(FileOrDirectoryArtifact.Create(output));
}
}
break;
}
},
constrainingPips));
}
