/// <inheritdoc /> public virtual DirectoryArtifact AddSealDirectory([NotNull] SealDirectory sealDirectory, PipId valuePip) { AddPip(sealDirectory); DirectoryArtifact artifactForNewSeal; if (sealDirectory.Kind == SealDirectoryKind.SharedOpaque) { Contract.Assume(sealDirectory.Directory.IsSharedOpaque); artifactForNewSeal = sealDirectory.Directory; } else { // For the regular dynamic case, the directory artifact is always // created with sealId 0. For other cases, we reserve it artifactForNewSeal = sealDirectory.Kind == SealDirectoryKind.Opaque ? DirectoryArtifact.CreateWithZeroPartialSealId(sealDirectory.DirectoryRoot) : SealDirectoryTable.ReserveDirectoryArtifact(sealDirectory); sealDirectory.SetDirectoryArtifact(artifactForNewSeal); } SealDirectoryTable.AddSeal(sealDirectory); ComputeStaticFingerprint(sealDirectory); return(artifactForNewSeal); }
public SealDirectoryStrings(SealDirectory sealDirectory, CachedGraph cachedGraph) { var pathTable = cachedGraph.Context.PathTable; var stringTable = cachedGraph.Context.StringTable; m_directoryRoot = sealDirectory.DirectoryRoot.ToString(pathTable).ToLowerInvariant(); { var patterns = sealDirectory .Patterns .Select(pattern => pattern.ToString(stringTable)); foreach (var pattern in patterns) { m_patterns.Add(pattern.ToLowerInvariant()); } } { var contents = sealDirectory .Contents .Select(fileArtifact => fileArtifact.Path.ToString(pathTable)); foreach (var content in contents) { m_contents.Add(content.ToLowerInvariant()); } } m_recursive = sealDirectory.Kind == SealDirectoryKind.SourceAllDirectories; }
private static bool TryScheduleSealDirectory(TestEnv env, AbsolutePath path, SealDirectoryKind partial, FileArtifact[] contents) { var pip = new SealDirectory( path, SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(contents, OrdinalFileArtifactComparer.Instance), kind: partial, provenance: env.CreatePipProvenance(StringId.Invalid), tags: ReadOnlyArray <StringId> .Empty, patterns: ReadOnlyArray <StringId> .Empty); DirectoryArtifact artifact = env.PipGraph.AddSealDirectory(pip, PipId.Invalid); bool succeeded = artifact.IsValid; if (succeeded) { FileArtifact[] actualContents = GetSealedDirectoryContents(env, artifact); XAssert.AreEqual(contents.Length, actualContents.Length, "Wrong number of contents sealed"); for (int i = 0; i < contents.Length; i++) { XAssert.IsTrue(contents[i] == actualContents[i], "Content artifact at position {0} mismatched", i); } } return(succeeded); }
public void ConsumeFilterPassingFile(bool topOnly) { FileArtifact source; if (topOnly) { source = FileArtifact.CreateSourceFile(SourceRootPath.Combine(Context.PathTable, "file.txt")); } else { var nestedDir = AbsolutePath.Create(Context.PathTable, Path.Combine(SourceRoot, "nested")); source = FileArtifact.CreateSourceFile(nestedDir.Combine(Context.PathTable, "file.txt")); } WriteSourceFile(source); var output = CreateOutputFileArtifact(ObjectRoot); SealDirectory sealedDirectory = CreateSourceSealDirectory(SourceRootPath, topOnly ? SealDirectoryKind.SourceTopDirectoryOnly : SealDirectoryKind.SourceAllDirectories, "*.txt", "*.cs"); DirectoryArtifact dir = PipGraphBuilder.AddSealDirectory(sealedDirectory); var builder = CreatePipBuilder(new Operation[] { Operation.ReadFile(source, doNotInfer: true), Operation.WriteFile(output), }); builder.AddInputDirectory(dir); SchedulePipBuilder(builder); RunScheduler().AssertSuccess(); }
/// <inheritdoc /> public override DirectoryArtifact AddSealDirectory([NotNull] SealDirectory sealDirectory, PipId valuePip) { base.AddSealDirectory(sealDirectory, valuePip); AddFileDependents(sealDirectory.Contents, sealDirectory); AddDirectoryDependents(sealDirectory.ComposedDirectories, sealDirectory); return(sealDirectory.Directory); }
public void TrustedAccessesAreBlockedIfPipDependsOnSourceSeal() { using (TestEnv env = TestEnv.CreateTestEnvWithPausedScheduler()) { var sourceSealedDirectory = new SealDirectory( env.SourceRoot, CollectionUtilities.EmptySortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer>(OrdinalFileArtifactComparer.Instance), outputDirectoryContents: CollectionUtilities.EmptySortedReadOnlyArray <DirectoryArtifact, OrdinalDirectoryArtifactComparer>(OrdinalDirectoryArtifactComparer.Instance), kind: SealDirectoryKind.SourceAllDirectories, provenance: env.CreatePipProvenance(StringId.Invalid), tags: ReadOnlyArray <StringId> .Empty, patterns: ReadOnlyArray <StringId> .Empty); DirectoryArtifact artifact = env.PipGraph.AddSealDirectory(sourceSealedDirectory, PipId.Invalid); Assert.True(artifact.IsValid); var pip = CreatePipBuilderWithTag(env, "test"); pip.AddInputDirectory(artifact); pip.Options |= Process.Options.TrustStaticallyDeclaredAccesses; var success = env.PipConstructionHelper.TryAddProcess(pip); Assert.False(success, "Finish should fail, a process depending on a source sealed directory is not allowed to trust declared accesses"); } }
public void ConsumeFilterNotPassingFile() { FileArtifact source = FileArtifact.CreateSourceFile(SourceRootPath.Combine(Context.PathTable, "file.txt")); WriteSourceFile(source); var output = CreateOutputFileArtifact(ObjectRoot); SealDirectory sealedDirectory = CreateSourceSealDirectory(SourceRootPath, SealDirectoryKind.SourceTopDirectoryOnly, "*.cs"); DirectoryArtifact dir = PipGraphBuilder.AddSealDirectory(sealedDirectory); var builder = CreatePipBuilder(new Operation[] { Operation.ReadFile(source, doNotInfer: true), Operation.WriteFile(output), }); builder.AddInputDirectory(dir); SchedulePipBuilder(builder); RunScheduler().AssertFailure(); AssertVerboseEventLogged(ProcessesLogEventId.PipProcessDisallowedFileAccess); AssertVerboseEventLogged(LogEventId.DependencyViolationMissingSourceDependency); AssertWarningEventLogged(LogEventId.ProcessNotStoredToCacheDueToFileMonitoringViolations); AssertErrorEventLogged(LogEventId.FileMonitoringError); }
public void IncrementalSchedulingIsRobustAgainstFileCreationMidBuild() { var directoryPath = CreateUniqueDirectory(ReadonlyRoot); var fileInsideDirectory = FileArtifact.CreateSourceFile(Combine(directoryPath, "fileToBeCreated")); FileArtifact outputA = CreateOutputFileArtifact(); SealDirectory sealedDirectory = CreateAndScheduleSealDirectory(directoryPath, SealDirectoryKind.Partial, fileInsideDirectory); var builderA = CreatePipBuilder(new Operation[] { Operation.Probe(fileInsideDirectory, doNotInfer: true), Operation.WriteFile(outputA, "Hello World A!") }); builderA.AddInputDirectory(sealedDirectory.Directory); var pipA = SchedulePipBuilder(builderA); FileArtifact outputB = CreateOutputFileArtifact(); FileArtifact rougeOutput = CreateOutputFileArtifact(directoryPath); var builderB = CreatePipBuilder(new Operation[] { Operation.ReadFile(outputA), Operation.WriteFile(outputB, "Hello World B!"), Operation.WriteFile(fileInsideDirectory, "Whatever", doNotInfer: true) }); builderB.AddUntrackedDirectoryScope(sealedDirectory.Directory); var pipB = SchedulePipBuilder(builderB); FileArtifact outputC = CreateOutputFileArtifact(); var builderC = CreatePipBuilder(new Operation[] { Operation.ReadFile(outputB), Operation.Probe(fileInsideDirectory, doNotInfer: true), Operation.WriteFile(outputC, "Hello World C!") }); builderC.AddInputDirectory(sealedDirectory.Directory); var pipC = SchedulePipBuilder(builderC); var result = RunScheduler().AssertScheduled( pipA.Process.PipId, pipB.Process.PipId, pipC.Process.PipId); // File creation blows away incremental scheduling state. RunScheduler() .AssertScheduled( pipA.Process.PipId, pipB.Process.PipId, pipC.Process.PipId) .AssertCacheMiss(pipA.Process.PipId, pipC.Process.PipId) .AssertCacheHit(pipB.Process.PipId); }
/// <summary> /// Adds a <see cref="SealDirectory"/> with a unique <see cref="BuildXL.Utilities.DirectoryArtifact"/> previously reserved with <see cref="ReserveDirectoryArtifact"/>. /// </summary> public void AddSeal(SealDirectory seal) { Contract.Requires(!IsReadOnly); Contract.Requires(seal != null); Contract.Requires(seal.IsInitialized, "Assign a directory artifact with SetDirectoryArtifact to the pip before adding it"); Contract.Requires(seal.PipId.IsValid, "SealDirectory pip must be added to the pip table before addition here"); m_pathToSealInfo.BackingSet.UpdateItem(new UpdateSealItem(this, seal)); }
private XElement GetSealDirectoryDetails(SealDirectory pip) { return(m_html.CreateBlock( "SealDirectory Pip Details", m_html.CreateEnumRow("Kind", pip.Kind), m_html.CreateRow("Scrub", pip.Scrub), m_html.CreateRow("DirectoryRoot", pip.Directory), m_html.CreateRow("Contents", pip.Contents))); }
/// <inheritdoc /> public DirectoryArtifact AddSealDirectory(SealDirectory sealDirectory, PipId valuePip) { Contract.Requires(sealDirectory != null); sealDirectory.PipId = GetPipId(); sealDirectory.SetDirectoryArtifact(DirectoryArtifact.CreateWithZeroPartialSealId(sealDirectory.DirectoryRoot)); Contract.Assume(sealDirectory.IsInitialized); m_pips.Enqueue(sealDirectory); return(sealDirectory.Directory); }
private string DepDescribe(Pip pip) { if (pip.PipType != PipType.SealDirectory) { return(pip.FormattedSemiStableHash); } SealDirectory sd = (SealDirectory)pip; return($"{pip.FormattedSemiStableHash}({sd.Contents.Length} files)"); }
public void TestSealDirectoryDump() { var sealPath = CreateOutputDirectoryArtifact(TemporaryDirectory); SealDirectory sealDirectory = CreateSealDirectory(sealPath.Path, SealDirectoryKind.Full, scrub: true, new[] { CreateSourceFile(), CreateSourceFile(), CreateSourceFile() }); sealDirectory.SetDirectoryArtifact(sealPath); PipGraphBuilder.AddSealDirectory(sealDirectory); RunAndAssertDumpPip(sealDirectory); }
/// <summary> /// Gets the outputs produced by a pip and calls an action /// </summary> public static bool ForEachOutput(Pip pip, Func <FileOrDirectoryArtifact, bool> outputAction, bool includeUncacheable) { bool result = true; switch (pip.PipType) { case PipType.CopyFile: CopyFile copyFile = (CopyFile)pip; result = outputAction(FileOrDirectoryArtifact.Create(copyFile.Destination)); break; case PipType.SealDirectory: SealDirectory sealDirectory = (SealDirectory)pip; result = outputAction(FileOrDirectoryArtifact.Create(sealDirectory.Directory)); break; case PipType.Process: Process process = (Process)pip; foreach (var output in process.FileOutputs) { if (includeUncacheable || output.CanBeReferencedOrCached()) { if (!outputAction(FileOrDirectoryArtifact.Create(output.ToFileArtifact()))) { return(false); } } } foreach (var output in process.DirectoryOutputs) { if (!outputAction(FileOrDirectoryArtifact.Create(output))) { return(false); } } break; case PipType.WriteFile: WriteFile writeFile = (WriteFile)pip; result = outputAction(FileOrDirectoryArtifact.Create(writeFile.Destination)); break; case PipType.Ipc: IpcPip ipcPip = (IpcPip)pip; result = outputAction(FileOrDirectoryArtifact.Create(ipcPip.OutputFile)); break; } return(result); }
private DirectoryArtifact GetSourceSeal(IMutablePipGraph pipGraph, AbsolutePath path) { var sealDirectory = new SealDirectory( path, contents: s_emptySealContents, kind: SealDirectoryKind.SourceAllDirectories, provenance: m_provenance, tags: ReadOnlyArray <StringId> .Empty, patterns: ReadOnlyArray <StringId> .Empty, scrub: false); return(pipGraph.AddSealDirectory(sealDirectory, default)); }
private DirectoryArtifact GetSourceSeal(PathTable pathTable, PipGraph.Builder pipGraph, string path) { var sealDirectory = new SealDirectory( AbsolutePath.Create(pathTable, path), contents: s_emptySealContents, kind: SealDirectoryKind.SourceAllDirectories, provenance: m_provenance, tags: ReadOnlyArray <StringId> .Empty, patterns: ReadOnlyArray <StringId> .Empty, scrub: false); return(pipGraph.AddSealDirectory(sealDirectory)); }
private void ReserveAndAddSeal(SealedDirectoryTable table, SealDirectory seal) { var directoryArtifact = table.ReserveDirectoryArtifact(seal); seal.SetDirectoryArtifact(directoryArtifact); table.AddSeal(seal); XAssert.IsTrue(seal.IsInitialized); XAssert.IsTrue(table.TryGetSealForDirectoryArtifact(directoryArtifact, out PipId pipId)); XAssert.AreEqual(seal.PipId, pipId); XAssert.IsTrue(directoryArtifact.IsValid); }
/// <summary> /// Generates the SealDirectoryPipDetails for a given Pip /// </summary> public SealDirectoryPipDetails GenerateSealDirectoryPipDetails(SealDirectory pip) { SealDirectoryPipDetails sealDirectoryPipDetails = new SealDirectoryPipDetails { Kind = pip.Kind, Scrub = pip.Scrub, DirectoryRoot = pip.Directory.Path.ToString(PathTable), Contents = pip.Contents.Select(x => x.Path.ToString(PathTable)).ToList() }; sealDirectoryPipDetails.Contents = sealDirectoryPipDetails.Contents.Any() ? sealDirectoryPipDetails.Contents : null; return(sealDirectoryPipDetails); }
private static SealDirectory CreateSeal(TestEnv env, AbsolutePath path, bool partial, FileArtifact[] contents) { var seal = new SealDirectory( path, SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(contents, OrdinalFileArtifactComparer.Instance), kind: partial ? SealDirectoryKind.Partial : SealDirectoryKind.Full, provenance: env.CreatePipProvenance(StringId.Invalid), tags: ReadOnlyArray <StringId> .Empty, patterns: ReadOnlyArray <StringId> .Empty); env.PipTable.Add(((PipGraph.Builder)(env.PipGraph)).MutableDataflowGraph.CreateNode().Value, seal); return(seal); }
/// <inheritdoc /> protected override void AddWeakFingerprint(IFingerprinter fingerprinter, SealDirectory sealDirectory) { Contract.Requires(fingerprinter != null); Contract.Requires(sealDirectory != null); base.AddWeakFingerprint(fingerprinter, sealDirectory); if (!ExcludeSemiStableHashOnFingerprintingSealDirectory && !sealDirectory.Kind.IsDynamicKind()) { // A statically sealed directory can exist as multiple different instances, e.g., one can have partially sealed directories with the same root and member set. // To distinguish those instances, we include the semi stable hash as part of the static fingerprint. fingerprinter.Add("SemiStableHash", sealDirectory.SemiStableHash); } }
/// <inheritdoc /> public DirectoryArtifact AddSealDirectory(SealDirectory sealDirectory, PipId valuePip) { Contract.Requires(sealDirectory != null); if (PipAlreadyReloaded(sealDirectory)) { DirectoryArtifact dir; return(m_reloadedSealDirectories.TryGetValue(sealDirectory.SemiStableHash, out dir) ? dir : DirectoryArtifact.Invalid); } else { return(m_builder.AddSealDirectory(sealDirectory)); } }
internal override void VisitDirectory(SealDirectory sealDirectory) { if (ProjectsByQualifier.TryGetValue(sealDirectory.Provenance.QualifierId, out var project)) { if (sealDirectory.Tags.Contains(Context.AssemblyDeploymentTag)) { SetOutputDirectory(project, sealDirectory.Directory.Path, OutputDirectoryType.AssemblyDeployment); } if (sealDirectory.Tags.Contains(Context.TestDeploymentTag)) { MakeTestProject(); SetOutputDirectory(project, sealDirectory.Directory.Path, OutputDirectoryType.TestDeployment); } } }
/// <summary> /// Reserves a new and unique <see cref="BuildXL.Utilities.DirectoryArtifact"/>. This should be assigned to a seal directory pip /// with <see cref="SealDirectory.SetDirectoryArtifact"/> before adding it with <see cref="AddSeal"/>. /// </summary> public DirectoryArtifact ReserveDirectoryArtifact(SealDirectory sealDirectory) { Contract.Requires(!IsReadOnly); // the following 2 preconditions are equivalent to `sealDirectory.IsInitialized == IsPatching`, but are split into 2 to give user friendly messages Contract.Requires(!sealDirectory.IsInitialized || IsPatching, "An initialized seal directory may only be added while IsPatching is true"); Contract.Requires(!IsPatching || sealDirectory.IsInitialized, "During patching, only initialized seal directory may be added"); if (sealDirectory.IsInitialized) { return(sealDirectory.Directory); } else { return(CreateDirectoryArtifactWithNewSealId(sealDirectory.DirectoryRoot, isSharedOpaque: false)); } }
/// <nodoc /> public bool TrySealDirectory( AbsolutePath directoryRoot, SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> contents, SortedReadOnlyArray <DirectoryArtifact, OrdinalDirectoryArtifactComparer> outputDirectorycontents, SealDirectoryKind kind, string[] tags, string description, string[] patterns, out DirectoryArtifact sealedDirectory, bool scrub = false) { Contract.Requires(directoryRoot.IsValid); Contract.Requires(contents.IsValid); Contract.Requires(outputDirectorycontents.IsValid); PipData usage = PipDataBuilder.CreatePipData(Context.StringTable, string.Empty, PipDataFragmentEscaping.NoEscaping, description != null ? new PipDataAtom[] { description } : new PipDataAtom[] { "'", directoryRoot, "' [", contents.Length.ToString(CultureInfo.InvariantCulture), " files - ", outputDirectorycontents.Length.ToString(CultureInfo.InvariantCulture), " output directories]" }); var pip = new SealDirectory( directoryRoot, contents, outputDirectorycontents, kind, CreatePipProvenance(usage), ToStringIds(tags), ToStringIds(patterns), scrub); if (PipGraph != null) { sealedDirectory = PipGraph.AddSealDirectory(pip, GetValuePipId()); if (!sealedDirectory.IsValid) { return(false); } } else { sealedDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(directoryRoot); } return(true); }
public void ExecutionRecordsReportedFileAccesses() { FileArtifact sourceFile = CreateSourceFile(); SealDirectory sourceDirectory = CreateAndScheduleSealDirectory(sourceFile.Path.GetParent(Context.PathTable), SealDirectoryKind.SourceAllDirectories); ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(sourceFile, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) }); builder.AddInputDirectory(sourceDirectory.Directory); builder.Options |= Process.Options.RequiresAdmin; ProcessWithOutputs process = SchedulePipBuilder(builder); RunScheduler().AssertSuccess(); RunScheduler().AssertCacheHit(process.Process.PipId); File.WriteAllText(ArtifactToString(sourceFile), Guid.NewGuid().ToString()); RunScheduler().AssertCacheMiss(process.Process.PipId); }
internal override void VisitDirectory(SealDirectory sealDirectory) { Project project; string friendlyQualifier = Context.QualifierTable.GetCanonicalDisplayString(sealDirectory.Provenance.QualifierId); if (ProjectsByQualifier.TryGetValue(friendlyQualifier, out project)) { if (sealDirectory.Tags.Contains(Context.AssemblyDeploymentTag)) { SetOutputDirectory(project, sealDirectory.Directory.Path, OutputDirectoryType.AssemblyDeployment); } if (sealDirectory.Tags.Contains(Context.TestDeploymentTag)) { MakeTestProject(); SetOutputDirectory(project, sealDirectory.Directory.Path, OutputDirectoryType.TestDeployment); } } }
private XElement GetSealDirectoryDetails(SealDirectory pip) { m_directoryContents.TryGetValue(pip.Directory, out var dynamicDirectoryContent); return(m_html.CreateBlock( "SealDirectory Pip Details", m_html.CreateEnumRow("Kind", pip.Kind), m_html.CreateRow("Scrub", pip.Scrub), m_html.CreateRow("DirectoryRoot", pip.Directory), m_html.CreateRow("DirectoryArtifact", SerializeDirectoryArtifact(pip.Directory)), m_html.CreateRow("ComposedDirectories", pip.ComposedDirectories), pip.ContentFilter.HasValue ? m_html.CreateRow("ContentFilter", contentFilterToString(pip.ContentFilter.Value)) : null, m_html.CreateRow("Contents", pip.Contents), pip.Kind == SealDirectoryKind.SharedOpaque ? m_html.CreateRow("Dynamic contents", dynamicDirectoryContent) : null)); string contentFilterToString(SealDirectoryContentFilter filter) { return($"{filter.Regex} (kind: {Enum.Format(typeof(SealDirectoryContentFilter.ContentFilterKind), filter.Kind, "f")})"); } }
/// <inheritdoc /> protected override void AddWeakFingerprint(IFingerprinter fingerprinter, SealDirectory sealDirectory) { Contract.Requires(fingerprinter != null); Contract.Requires(sealDirectory != null); base.AddWeakFingerprint(fingerprinter, sealDirectory); // For non-composite shared opaque directories, contents and composed directories are always empty, and therefore the static fingerprint // is not strong enough, i.e. multiple shared opaques can share the same directory root. So in this case we need to add the fingerprint of the producer if (sealDirectory.Kind == SealDirectoryKind.SharedOpaque && !sealDirectory.IsComposite) { DirectoryArtifact directory = sealDirectory.Directory; fingerprinter.Add(directory, m_directoryProducerFingerprintLookup(directory).Hash); } if (!ExcludeSemiStableHashOnFingerprintingSealDirectory && !sealDirectory.Kind.IsDynamicKind()) { // A statically sealed directory can exist as multiple different instances, e.g., one can have partially sealed directories with the same root and member set. // To distinguish those instances, we include the semi stable hash as part of the static fingerprint. fingerprinter.Add("SemiStableHash", sealDirectory.SemiStableHash); } }
/// <summary> /// Gets the outputs produced by a pip and calls an action /// </summary> protected static void ForEachOutput <TState>(TState state, Pip pip, Action <TState, FileOrDirectoryArtifact> outputAction) { switch (pip.PipType) { case PipType.CopyFile: CopyFile copyFile = (CopyFile)pip; outputAction(state, FileOrDirectoryArtifact.Create(copyFile.Destination)); break; case PipType.Process: Process process = (Process)pip; foreach (var output in process.FileOutputs) { outputAction(state, FileOrDirectoryArtifact.Create(output.ToFileArtifact())); } foreach (var output in process.DirectoryOutputs) { outputAction(state, FileOrDirectoryArtifact.Create(output)); } break; case PipType.WriteFile: WriteFile writeFile = (WriteFile)pip; outputAction(state, FileOrDirectoryArtifact.Create(writeFile.Destination)); break; case PipType.SealDirectory: SealDirectory sealDirectory = (SealDirectory)pip; outputAction(state, FileOrDirectoryArtifact.Create(sealDirectory.Directory)); break; case PipType.Ipc: IpcPip ipcPip = (IpcPip)pip; outputAction(state, FileOrDirectoryArtifact.Create(ipcPip.OutputFile)); break; } }
internal virtual void VisitDirectory(SealDirectory sealDirectory) { }