/// <summary> /// Gets the outputs produced by a pip and calls an action /// </summary> public static bool ForEachOutput(Pip pip, Func <FileOrDirectoryArtifact, bool> outputAction, bool includeUncacheable) { bool result = true; switch (pip.PipType) { case PipType.CopyFile: CopyFile copyFile = (CopyFile)pip; result = outputAction(FileOrDirectoryArtifact.Create(copyFile.Destination)); break; case PipType.SealDirectory: SealDirectory sealDirectory = (SealDirectory)pip; result = outputAction(FileOrDirectoryArtifact.Create(sealDirectory.Directory)); break; case PipType.Process: Process process = (Process)pip; foreach (var output in process.FileOutputs) { if (includeUncacheable || output.CanBeReferencedOrCached()) { if (!outputAction(FileOrDirectoryArtifact.Create(output.ToFileArtifact()))) { return(false); } } } foreach (var output in process.DirectoryOutputs) { if (!outputAction(FileOrDirectoryArtifact.Create(output))) { return(false); } } break; case PipType.WriteFile: WriteFile writeFile = (WriteFile)pip; result = outputAction(FileOrDirectoryArtifact.Create(writeFile.Destination)); break; case PipType.Ipc: IpcPip ipcPip = (IpcPip)pip; result = outputAction(FileOrDirectoryArtifact.Create(ipcPip.OutputFile)); break; } return(result); }
/// <summary> /// Gets the outputs produced by a pip and calls an action /// </summary> protected static void ForEachOutput <TState>(TState state, Pip pip, Action <TState, FileOrDirectoryArtifact> outputAction) { switch (pip.PipType) { case PipType.CopyFile: CopyFile copyFile = (CopyFile)pip; outputAction(state, FileOrDirectoryArtifact.Create(copyFile.Destination)); break; case PipType.Process: Process process = (Process)pip; foreach (var output in process.FileOutputs) { outputAction(state, FileOrDirectoryArtifact.Create(output.ToFileArtifact())); } foreach (var output in process.DirectoryOutputs) { outputAction(state, FileOrDirectoryArtifact.Create(output)); } break; case PipType.WriteFile: WriteFile writeFile = (WriteFile)pip; outputAction(state, FileOrDirectoryArtifact.Create(writeFile.Destination)); break; case PipType.SealDirectory: SealDirectory sealDirectory = (SealDirectory)pip; outputAction(state, FileOrDirectoryArtifact.Create(sealDirectory.Directory)); break; case PipType.Ipc: IpcPip ipcPip = (IpcPip)pip; outputAction(state, FileOrDirectoryArtifact.Create(ipcPip.OutputFile)); break; } }
public void AddRealPath(string fullPath, PathExistence existence) { if (existence == PathExistence.Nonexistent) { return; } var path = AbsolutePath.Create(m_pathTable, fullPath); FileOrDirectoryArtifact member = FileOrDirectoryArtifact.Invalid; while (path.IsValid) { if (existence == PathExistence.ExistsAsFile) { m_files.Add(path); member = new FileArtifact(path); } else { var members = m_directories.GetOrAdd(path, p => new HashSet <FileOrDirectoryArtifact>()); if (member.IsValid) { if (!members.Add(member)) { return; } } member = DirectoryArtifact.CreateWithZeroPartialSealId(path); } existence = PathExistence.ExistsAsDirectory; path = path.GetParent(m_pathTable); } }
/// <summary> /// Matches the string representation of <see cref="FileOrDirectoryArtifact"/> used by the fingerprint store /// when serializing to JSON. /// </summary> private string ArtifactToPrint(FileOrDirectoryArtifact artifact) { return(Expander.ExpandPath(Context.PathTable, artifact.Path).ToLowerInvariant().Replace(@"\", @"\\")); }
/*** TO STRING HELPER FUNCTIONS ***/ /// <summary> /// Converts FileOrDirectoryArtifact to string /// </summary> public string FileOrDirectoryToString(FileOrDirectoryArtifact fileArtifact) { return(fileArtifact.Path.ToString(PathTable)); }
/// <summary> /// Creates a create symbolic link operation /// </summary> /// <remarks> /// This method is for testing symlinks whose target either /// (a) cannot be represented using FileArtifact/AbsolutePath (e.g., path contains an illegal char), or /// (b) should not be represented using these structs (e.g., target's path is a relative path) /// </remarks> public static Operation CreateSymlink(FileOrDirectoryArtifact linkPath, string target, SymbolicLinkFlag symLinkFlag, bool doNotInfer = false) { return(new Operation(Type.CreateSymlink, linkPath: linkPath, additionalArgs: target, symLinkFlag: symLinkFlag, doNotInfer: doNotInfer)); }
/// <summary> /// Creates a probe operation /// </summary> public static Operation Probe(FileOrDirectoryArtifact path, bool doNotInfer = false) { return(new Operation(Type.Probe, path, doNotInfer: doNotInfer)); }
/// <summary> /// Creates a create directory operation (uses WinAPI) /// </summary> public static Operation CreateDir(FileOrDirectoryArtifact path, bool doNotInfer = false) { return(new Operation(Type.CreateDir, path, doNotInfer: doNotInfer)); }
private bool TryScheduleIpcPip(Context context, ObjectLiteral obj, bool allowUndefinedTargetService, bool isServiceFinalization, out FileArtifact outputFile, out PipId pipId) { // IpcClientInfo IIpcMoniker moniker = Converter.ExtractRef <IIpcMoniker>(obj, m_ipcSendMoniker, allowUndefined: false); int? numRetries = Converter.ExtractNumber(obj, m_ipcSendMaxConnectRetries, allowUndefined: true); int? retryDelayMillis = Converter.ExtractNumber(obj, m_ipcSendConnectRetryDelayMillis, allowUndefined: true); var clientConfig = new ClientConfig(numRetries, retryDelayMillis); var ipcClientInfo = new IpcClientInfo(moniker.ToStringId(context.StringTable), clientConfig); // target service pip PipId?servicePipId = Converter.ExtractValue <PipId>(obj, m_ipcSendTargetServicePip, allowUndefined: allowUndefinedTargetService); // arguments PipData arguments; ReadOnlyArray <FileArtifact> fileDependencies; ReadOnlyArray <DirectoryArtifact> directoryDependencies; using (var ipcProcessBuilder = ProcessBuilder.Create(context.PathTable, context.FrontEndContext.GetPipDataBuilder())) { // process arguments ArrayLiteral argumentsArrayLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendMessageBody); TransformerExecuteArgumentsProcessor.ProcessArguments(context, ipcProcessBuilder, argumentsArrayLiteral); // input file dependencies var dependenciesArray = Converter.ExtractArrayLiteral(obj, m_ipcSendDependencies, allowUndefined: true); if (dependenciesArray != null) { for (int i = 0; i < dependenciesArray.Length; i++) { ProcessImplicitDependency(ipcProcessBuilder, dependenciesArray[i], convContext: new ConversionContext(pos: i, objectCtx: dependenciesArray)); } } arguments = ipcProcessBuilder.ArgumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules); fileDependencies = ipcProcessBuilder.GetInputFilesSoFar(); directoryDependencies = ipcProcessBuilder.GetInputDirectoriesSoFar(); } // output AbsolutePath output = Converter.ExtractPath(obj, m_ipcSendOutputFile, allowUndefined: true); if (!output.IsValid) { output = context.GetPipConstructionHelper().GetUniqueObjectDirectory(m_ipcObjectFolderName).Path.Combine(context.PathTable, m_ipcOutputFileName); } // tags string[] tags = null; var tagsArray = Converter.ExtractArrayLiteral(obj, m_executeTags, allowUndefined: true); if (tagsArray != null && tagsArray.Count > 0) { tags = new string[tagsArray.Count]; for (int i = 0; i < tagsArray.Count; i++) { tags[i] = Converter.ExpectString(tagsArray[i], context: new ConversionContext(pos: i, objectCtx: tagsArray)); } } // skip materialization for files FileOrDirectoryArtifact[] skipMaterializationArtifacts = CollectionUtilities.EmptyArray <FileOrDirectoryArtifact>(); ArrayLiteral skipMaterializationLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendLazilyMaterializedDependencies, allowUndefined: true); if (skipMaterializationLiteral != null) { skipMaterializationArtifacts = new FileOrDirectoryArtifact[skipMaterializationLiteral.Length]; for (int i = 0; i < skipMaterializationLiteral.Length; i++) { Converter.ExpectFileOrStaticDirectory( skipMaterializationLiteral[i], out var fileArtifact, out var staticDirectory, context: new ConversionContext(pos: i, objectCtx: skipMaterializationLiteral)); Contract.Assert(fileArtifact.IsValid ^ staticDirectory != null); skipMaterializationArtifacts[i] = fileArtifact.IsValid ? FileOrDirectoryArtifact.Create(fileArtifact) : FileOrDirectoryArtifact.Create(staticDirectory.Root); } } // must run on master var mustRunOnMaster = Converter.ExtractOptionalBoolean(obj, m_ipcSendMustRunOnMaster) == true; outputFile = FileArtifact.CreateOutputFile(output); // create IPC pip and add it to the graph bool result = context.GetPipConstructionHelper().TryAddIpc( ipcClientInfo, arguments, outputFile, servicePipDependencies: servicePipId != null ? ReadOnlyArray <PipId> .From(new[] { servicePipId.Value }) : ReadOnlyArray <PipId> .Empty, fileDependencies: fileDependencies, directoryDependencies: directoryDependencies, skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .FromWithoutCopy(skipMaterializationArtifacts), isServiceFinalization: isServiceFinalization, mustRunOnMaster: mustRunOnMaster, tags: tags, out var ipcPip); pipId = ipcPip.PipId; return(result); }
private IExpression Generate(FileOrDirectoryArtifact artifact) { return(Generate(artifact.Path, artifact.IsFile ? "f" : "d")); }
/// <summary> /// Creates a create directory operation (uses WinAPI) /// </summary> public static Operation CreateDir(FileOrDirectoryArtifact path, bool doNotInfer = false, string additionalArgs = null) { return(new Operation(Type.CreateDir, path, doNotInfer: doNotInfer, additionalArgs: additionalArgs)); }
public string ArtifactToString(FileOrDirectoryArtifact file, PathTable pathTable = null) { return(file.Path.ToString(pathTable ?? Context.PathTable)); }
/// <summary> /// Gets the inputs consumed by a pip and calls an action /// </summary> public static bool ForEachInput( Pip pip, Func <FileOrDirectoryArtifact, bool> inputAction, bool includeLazyInputs, Func <FileOrDirectoryArtifact, bool> overrideLazyInputAction = null) { // NOTE: Lazy inputs must be processed AFTER regular inputs // This behavior is required by FileContentManager.PopulateDepdencies bool result = true; switch (pip.PipType) { case PipType.CopyFile: CopyFile copyFile = (CopyFile)pip; result = inputAction(FileOrDirectoryArtifact.Create(copyFile.Source)); break; case PipType.Process: Process process = (Process)pip; foreach (var input in process.Dependencies) { if (!inputAction(FileOrDirectoryArtifact.Create(input))) { return(false); } } foreach (var input in process.DirectoryDependencies) { if (!inputAction(FileOrDirectoryArtifact.Create(input))) { return(false); } } break; case PipType.SealDirectory: SealDirectory sealDirectory = (SealDirectory)pip; foreach (var input in sealDirectory.Contents) { if (!inputAction(FileOrDirectoryArtifact.Create(input))) { return(false); } } break; case PipType.Ipc: IpcPip ipcPip = (IpcPip)pip; foreach (var input in ipcPip.FileDependencies) { if (!inputAction(FileOrDirectoryArtifact.Create(input))) { return(false); } } foreach (var input in ipcPip.DirectoryDependencies) { if (!inputAction(FileOrDirectoryArtifact.Create(input))) { return(false); } } if (includeLazyInputs) { overrideLazyInputAction = overrideLazyInputAction ?? inputAction; foreach (var input in ipcPip.LazilyMaterializedDependencies) { if (!overrideLazyInputAction(input)) { return(false); } } } break; } return(result); }
/// <inheritdoc/> public override IReadOnlySet <FileOrDirectoryArtifact> FilterOutputsCore(IPipFilterContext context, bool negate = false, IList <PipId> constrainingPips = null) { // First we collect all matching seal directories HashSet <DirectoryArtifact> directories = ParallelProcessAllOutputs <DirectoryArtifact>( context, (pipId, localDirectories) => { if (context.GetPipType(pipId) == PipType.SealDirectory) { SealDirectory sd = (SealDirectory)context.HydratePip(pipId); foreach (var item in sd.Contents) { if (PathMatches(item.Path, context.PathTable)) { localDirectories.Add(sd.Directory); break; } } switch (sd.Kind) { case SealDirectoryKind.SourceAllDirectories: case SealDirectoryKind.Opaque: case SealDirectoryKind.SharedOpaque: if (DirectoryPathMatches(sd.DirectoryRoot, false, context.PathTable)) { localDirectories.Add(sd.Directory); } break; case SealDirectoryKind.SourceTopDirectoryOnly: if (DirectoryPathMatches(sd.DirectoryRoot, true, context.PathTable)) { localDirectories.Add(sd.Directory); } break; } } }); // Now look at all pips, checking if their input match one of the files or matching DirectoryArtifacts return(ParallelProcessAllOutputs <FileOrDirectoryArtifact>( context, (pipId, localOutputs) => { switch (context.GetPipType(pipId)) { case PipType.CopyFile: CopyFile cf = (CopyFile)context.HydratePip(pipId); if (PathMatches(cf.Source, context.PathTable) ^ negate) { localOutputs.Add(FileOrDirectoryArtifact.Create(cf.Destination)); } break; case PipType.Process: Process proc = (Process)context.HydratePip(pipId); bool processMatches = PathMatches(proc.Executable, context.PathTable) || MatchesDependencies(context, proc) || MatchesDirectoryDependencies(proc, directories); if (processMatches ^ negate) { // TODO: If only directory dependencies matched, only include outputs from those directories foreach (var output in proc.FileOutputs) { localOutputs.Add(FileOrDirectoryArtifact.Create(output.ToFileArtifact())); } foreach (var output in proc.DirectoryOutputs) { localOutputs.Add(FileOrDirectoryArtifact.Create(output)); } } break; } }, constrainingPips)); }