public void AbsentProbeOnOutputFileIsBlocked() { var sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, SharedOpaqueDirectoryRoot); FileArtifact source = CreateSourceFile(sharedOpaqueDirPath); // Create a pip that creates and deletes the source file under a shared opaque var sharedOpaqueDirArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath); var deletePipBuilder = CreatePipBuilder(new[] { Operation.WriteFile(source.CreateNextWrittenVersion(), doNotInfer: true), Operation.DeleteFile(source.CreateNextWrittenVersion(), doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) // dummy output }); deletePipBuilder.AddOutputDirectory(sharedOpaqueDirArtifact, SealDirectoryKind.SharedOpaque); var deletePip = SchedulePipBuilder(deletePipBuilder); // Creates a pip that probes the (deleted) source file (undeclared). // Make the reader depend on the dummy output of the writer to ensure read happens after write ScheduleProcessWithUndeclaredReads(source, fileDependencies: deletePip.ProcessOutputs.GetOutputFiles(), probeInsteadOfRead: true); RunScheduler().AssertFailure(); AssertErrorEventLogged(LogEventId.DependencyViolationWriteInUndeclaredSourceRead); }
/// <inheritdoc /> public virtual DirectoryArtifact AddSealDirectory([NotNull] SealDirectory sealDirectory, PipId valuePip) { AddPip(sealDirectory); DirectoryArtifact artifactForNewSeal; if (sealDirectory.Kind == SealDirectoryKind.SharedOpaque) { Contract.Assume(sealDirectory.Directory.IsSharedOpaque); artifactForNewSeal = sealDirectory.Directory; } else { // For the regular dynamic case, the directory artifact is always // created with sealId 0. For other cases, we reserve it artifactForNewSeal = sealDirectory.Kind == SealDirectoryKind.Opaque ? DirectoryArtifact.CreateWithZeroPartialSealId(sealDirectory.DirectoryRoot) : SealDirectoryTable.ReserveDirectoryArtifact(sealDirectory); sealDirectory.SetDirectoryArtifact(artifactForNewSeal); } SealDirectoryTable.AddSeal(sealDirectory); ComputeStaticFingerprint(sealDirectory); return(artifactForNewSeal); }
public void ValidateCachingTreatDirectoryAsAbsentFileOnHashingInputContent() { Configuration.Schedule.TreatDirectoryAsAbsentFileOnHashingInputContent = true; // start with absent /dir DirectoryArtifact dir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory()); FileArtifact fileVersionOfDir = new FileArtifact(dir.Path); Process pip = CreateAndSchedulePipBuilder(new Operation[] { Operation.Probe(fileVersionOfDir), Operation.WriteFile(CreateOutputFileArtifact()) }).Process; RunScheduler().AssertCacheMiss(pip.PipId); RunScheduler().AssertCacheHit(pip.PipId); // /dir will be hashed as an absent file on every run, so the pip fingerprint remains the same // Create /dir Directory.CreateDirectory(ArtifactToString(dir)); RunScheduler().AssertCacheHit(pip.PipId); // Create nested /dir/file CreateSourceFile(ArtifactToString(dir)); RunScheduler().AssertCacheHit(pip.PipId); }
/// <inheritdoc/> protected override void ProcessOutputs(JavaScriptProject project, ProcessBuilder processBuilder) { base.ProcessOutputs(project, processBuilder); // This makes sure the folder the user profile is pointing to gets actually created processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(UserProfile(project, PathTable)), SealDirectoryKind.SharedOpaque); }
public void RunSingleProcessWithSharedOpaqueOutputLogging() { Configuration.Schedule.UnsafeLazySODeletion = true; var sharedOpaqueDir = Path.Combine(ObjectRoot, "partialDir"); var sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir); var sharedOpaqueDirectoryArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath); var outputInSharedOpaque = CreateOutputFileArtifact(sharedOpaqueDir); var source = CreateSourceFile(); var builder = CreatePipBuilder(new[] { Operation.WriteFile(outputInSharedOpaque, content: "sod-out", doNotInfer: true) }); builder.AddOutputDirectory(sharedOpaqueDirectoryArtifact, SealDirectoryKind.SharedOpaque); builder.Options |= Process.Options.RequiresAdmin; var pip = SchedulePipBuilder(builder); // run once and assert success var result = RunScheduler().AssertSuccess(); // check that shared opaque outputs have been logged in the sideband file AssertWritesJournaled(result, pip, outputInSharedOpaque); // run again, assert cache hit, assert sideband files were used to postpone scrubbing RunScheduler().AssertCacheHit(pip.Process.PipId); AssertWritesJournaled(result, pip, outputInSharedOpaque); AssertInformationalEventLogged(EventId.PostponingDeletionOfSharedOpaqueOutputs, count: 1); }
/// <summary> /// Adds all known output directories /// </summary> /// <remarks> /// The project root (excluding node_modules) is always considered an output directory /// </remarks> protected virtual void ProcessOutputs(JavaScriptProject project, ProcessBuilder processBuilder) { // Each project is automatically allowed to write anything under its project root processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder), SealDirectoryKind.SharedOpaque); if (m_resolverSettings.BlockWritesUnderNodeModules == true) { // There shouldn't be any writes under node_modules. So exclude it explicitly, since that also avoids a usually expensive enumeration // under node_modules when scrubbing. processBuilder.AddOutputDirectoryExclusion(project.NodeModulesFolder(m_context.PathTable)); } // Some projects share their temp folder across their build scripts (e.g. build and test) // So we cannot make them share the temp folder with the infrastructure we have today // (even though not impossible to fix, we could allow temp directories among pips that are part // of the same depedency chain and eagerly delete the folder every time a pip finishes) processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(project.TempFolder), SealDirectoryKind.SharedOpaque); // Add all the additional output directories that the graph knows about foreach (var outputDirectory in project.OutputDirectories) { processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(outputDirectory), SealDirectoryKind.SharedOpaque); } // Add additional output directories configured in the main config file AddAdditionalOutputDirectories(processBuilder, project.ProjectFolder); }
private static EvaluationResult GetBuildEngineDirectoryToBeDeprecated(Context context, ModuleLiteral env, EvaluationStackFrame args) { var pathTable = context.FrontEndContext.PathTable; var executingEnginePath = AbsolutePath.Create(pathTable, AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly())); return(EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(executingEnginePath.GetParent(pathTable)))); }
protected ProcessWithOutputs CreateAndScheduleSharedOpaqueProducer(string sharedOpaqueDir, FileArtifact fileToProduceStatically, FileArtifact sourceFileToRead, params KeyValuePair <FileArtifact, string>[] filesAndContentToProduceDynamically) { AbsolutePath sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir); var sharedOpaqueDirectoryArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath); var operations = new List <Operation>(); if (fileToProduceStatically.IsValid) { operations.Add(Operation.WriteFile(fileToProduceStatically)); } if (sourceFileToRead.IsValid) { operations.Add(Operation.ReadFile(sourceFileToRead)); } foreach (var fac in filesAndContentToProduceDynamically) { operations.Add(Operation.WriteFile(fac.Key, content: fac.Value, doNotInfer: true)); } var builder = CreatePipBuilder(operations); builder.AddOutputDirectory(sharedOpaqueDirectoryArtifact, SealDirectoryKind.SharedOpaque); return(SchedulePipBuilder(builder)); }
public void ToolEnumeratingFile() { var sealDirectoryPath = CreateUniqueDirectory(ObjectRoot); var path = sealDirectoryPath.ToString(Context.PathTable); var nestedFile1 = CreateSourceFile(path); var nestedFile2 = CreateUniquePath("nonExistentFile", path); DirectoryArtifact dir = SealDirectory(sealDirectoryPath, SealDirectoryKind.SourceAllDirectories); Configuration.Sandbox.UnsafeSandboxConfigurationMutable.UnexpectedFileAccessesAreErrors = false; var ops = new Operation[] { Operation.EnumerateDir(DirectoryArtifact.CreateWithZeroPartialSealId(nestedFile2), doNotInfer: true, enumeratePattern: "*"), Operation.ReadFile(nestedFile1, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()), // This write will be allowed because UnexpectedFileAccessesAreErrors = false. // But this will make the above reporting of enumerating non-existing file as enumerating existing file. Operation.WriteFile(FileArtifact.CreateOutputFile(nestedFile2), doNotInfer: true) }; var builder = CreatePipBuilder(ops); builder.AddInputDirectory(dir); Process pip = SchedulePipBuilder(builder).Process; var result = RunScheduler().AssertSuccess(); AssertWarningEventLogged(EventId.ProcessNotStoredToCacheDueToFileMonitoringViolations, count: 2); AssertWarningEventLogged(EventId.FileMonitoringWarning, count: 1); }
public void ValidateBasicCacheMissCounters() { AbsolutePath.TryCreate(Context.PathTable, ReadonlyRoot, out var readonlyRootPath); var readonlyRootDir = DirectoryArtifact.CreateWithZeroPartialSealId(readonlyRootPath); var childDir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(ReadonlyRoot)); // Enumerate /readonlyroot and /readonlyroot/childDir Process enumeratorPip = CreateAndSchedulePipBuilder(new Operation[] { Operation.EnumerateDir(childDir), Operation.WriteFile(CreateOutputFileArtifact()) }).Process; Process writerPip = CreateAndSchedulePipBuilder(new Operation[] { Operation.WriteFile(CreateOutputFileArtifact()) }).Process; ScheduleRunResult result1 = RunScheduler(); result1.AssertCacheMiss(enumeratorPip.PipId, writerPip.PipId); result1.AssertNumWeakFingerprintMisses(2); // Create /childDir/nestedFile, causing strong fingerprint miss FileArtifact nestedFile = CreateSourceFile(ArtifactToString(childDir)); ScheduleRunResult result2 = RunScheduler(); result2.AssertCacheHit(writerPip.PipId); // Weak fingerprint hit on both pips result2.AssertNumWeakFingerprintMisses(0); // Strong fingerprint miss on enumeratorPip result2.AssertCacheMiss(enumeratorPip.PipId); result2.AssertNumStrongFingerprintMisses(1); }
protected ProcessBuilder CreateTempDirProcessBuilder(Operation[] ops, int tempArtifactType, AbsolutePath tempRootPath, AbsolutePath tempOut) { var pipBuilder = CreatePipBuilder(ops); switch (tempArtifactType) { case TempArtifactType.AdditionalTempDirectory: var temp = new AbsolutePath[pipBuilder.AdditionalTempDirectories.Length + 1]; for (int i = 0; i < pipBuilder.AdditionalTempDirectories.Length; i++) { temp[i] = pipBuilder.AdditionalTempDirectories[i]; } temp[pipBuilder.AdditionalTempDirectories.Length] = tempRootPath; pipBuilder.AdditionalTempDirectories = ReadOnlyArray <AbsolutePath> .FromWithoutCopy(temp); break; case TempArtifactType.TempDirectory: pipBuilder.SetTempDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(tempRootPath)); break; case TempArtifactType.TempFile: pipBuilder.AddOutputFile(tempOut, FileExistence.Temporary); break; default: XAssert.Fail("A test using a temporary file or directory didn't specify the temp artifact correctly."); return(null); } return(pipBuilder); }
public void InconsistentTempDirectoriesCreation(bool ensureTempDirectoriesCreation) { Configuration.Sandbox.EnsureTempDirectoriesExistenceBeforePipExecution = ensureTempDirectoriesCreation; AbsolutePath additionalTempDirectory = TempRootPath.Combine( Context.PathTable, nameof(InconsistentTempDirectoriesCreation) + "_" + ensureTempDirectoriesCreation); var pipBuilder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.CreateDir(DirectoryArtifact.CreateWithZeroPartialSealId(additionalTempDirectory), additionalArgs: "--failIfExists"), Operation.WriteFile(CreateOutputFileArtifact()) }); pipBuilder.EnableTempDirectory(); pipBuilder.AdditionalTempDirectories = ReadOnlyArray <AbsolutePath> .FromWithoutCopy(new AbsolutePath[] { additionalTempDirectory }); var processWithOutputs = SchedulePipBuilder(pipBuilder); var result = RunScheduler(); if (ensureTempDirectoriesCreation) { result.AssertFailure(); AssertErrorEventLogged(EventId.PipProcessError, 1); } else { result.AssertSuccess(); } }
public void ValidateCachingUntrackedPathInput(bool declareDependency, bool inputUnderUntrackedScope) { DirectoryArtifact dir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(SourceRoot)); FileArtifact nestedFile = CreateSourceFile(ArtifactToString(dir)); // Take input from an untracked path /dir/nestedFile var ops = new Operation[] { Operation.ReadFile(nestedFile, doNotInfer: !declareDependency), Operation.WriteFile(CreateOutputFileArtifact()) }; var builder = CreatePipBuilder(ops); if (inputUnderUntrackedScope) { builder.AddUntrackedDirectoryScope(dir); } else { builder.AddUntrackedFile(nestedFile); } Process pip = SchedulePipBuilder(builder).Process; RunScheduler().AssertCacheMiss(pip.PipId); RunScheduler().AssertCacheHit(pip.PipId); // Modify untracked input file /dir/nestedFile File.WriteAllText(ArtifactToString(nestedFile), "nestedFile"); var runSchedulerResult = RunScheduler(); if (!declareDependency || !inputUnderUntrackedScope) { runSchedulerResult.AssertCacheHit(pip.PipId); } else { runSchedulerResult.AssertCacheMiss(pip.PipId); } ResetPipGraphBuilder(); // Validate cache miss if the process removes untracked path declarations pip = CreateAndSchedulePipBuilder(ops).Process; runSchedulerResult = RunScheduler().AssertCacheMissWithoutAssertingSuccess(pip.PipId); if (!declareDependency) { runSchedulerResult.AssertPipResultStatus((pip.PipId, PipResultStatus.Failed)); AssertWarningEventLogged(SchedulerLogEventId.ProcessNotStoredToCacheDueToFileMonitoringViolations); AssertErrorEventLogged(SchedulerLogEventId.FileMonitoringError); } else { runSchedulerResult.AssertSuccess(); } }
public void PreserveOutputsOpaqueTestWithWhitelist() { Configuration.Sandbox.UnsafeSandboxConfigurationMutable.PreserveOutputs = PreserveOutputsMode.Enabled; var input = CreateSourceFile(); var opaquePreservedPath = AbsolutePath.Create(Context.PathTable, Path.Combine(ObjectRoot, "opaquePreservedDir")); var outputUnderPreservedOpaque = CreateOutputFileArtifact(opaquePreservedPath); var createdDirectoryUnderPreservedOpaque = DirectoryArtifact.CreateWithZeroPartialSealId(opaquePreservedPath.Combine(Context.PathTable, "CreatedDir")); var opaqueUnpreservedPath = AbsolutePath.Create(Context.PathTable, Path.Combine(ObjectRoot, "opaqueUnpreservedDir")); var outputUnderUnpreservedOpaque = CreateOutputFileArtifact(opaqueUnpreservedPath); var createdDirectoryUnderUnpreservedOpaque = DirectoryArtifact.CreateWithZeroPartialSealId(opaqueUnpreservedPath.Combine(Context.PathTable, "CreatedDir")); var builder = CreatePipBuilder(new Operation[] { Operation.ReadFile(input), Operation.WriteFile(outputUnderPreservedOpaque, CONTENT, doNotInfer: true), Operation.CreateDir(createdDirectoryUnderPreservedOpaque, doNotInfer: true), Operation.WriteFile(outputUnderUnpreservedOpaque, CONTENT, doNotInfer: true), Operation.CreateDir(createdDirectoryUnderUnpreservedOpaque, doNotInfer: true), }); builder.AddOutputDirectory(opaquePreservedPath); builder.AddOutputDirectory(opaqueUnpreservedPath); builder.Options |= Process.Options.AllowPreserveOutputs; builder.PreserveOutputWhitelist = ReadOnlyArray <AbsolutePath> .FromWithoutCopy(opaquePreservedPath); var processAndOutputs = SchedulePipBuilder(builder); // No cache hit string outputContents = RunSchedulerAndGetOutputContents(outputUnderPreservedOpaque, cacheHitAssert: false, id: processAndOutputs.Process.PipId); XAssert.AreEqual(CONTENT, outputContents); XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnderUnpreservedOpaque))); // Change input ModifyFile(input); // No cache hit outputContents = RunSchedulerAndGetOutputContents(outputUnderPreservedOpaque, cacheHitAssert: false, id: processAndOutputs.Process.PipId); // As the opaque output is preserved, the pip appended the existing file. XAssert.AreEqual(CONTENT_TWICE, outputContents); // For the file under unpreserved opaque directory, the file was created, so we did not append. XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnderUnpreservedOpaque))); // Cache hit outputContents = RunSchedulerAndGetOutputContents(outputUnderPreservedOpaque, cacheHitAssert: true, id: processAndOutputs.Process.PipId); XAssert.IsTrue(Directory.Exists(createdDirectoryUnderPreservedOpaque.Path.ToString(Context.PathTable)), "Empty directory under preserved opaque should have existed."); // Incremental scheduling doesn't replay the pip from cache and just leaves the filesystem as-is if (!Configuration.Schedule.GraphAgnosticIncrementalScheduling && !Configuration.Schedule.GraphAgnosticIncrementalScheduling) { XAssert.IsFalse(Directory.Exists(createdDirectoryUnderUnpreservedOpaque.Path.ToString(Context.PathTable)), "Empty directory under non-preserved opaque should not exist."); } // The appended file (CONTENT_TWICE) should remain the same. XAssert.AreEqual(CONTENT_TWICE, outputContents); XAssert.AreEqual(CONTENT, File.ReadAllText(ArtifactToString(outputUnderUnpreservedOpaque))); }
public void CompositeContentIsProperlyMaterialized() { // This test verifies that BXL properly materializes composite shared opaques // (i.e., the files are materialized because we need to materialize the composite opaque // and not because we somehow materialized the original shared opaque directory). var root = Path.Combine(ObjectRoot, "root"); var rootPath = AbsolutePath.Create(Context.PathTable, root); var sharedOpaqueDir = Path.Combine(ObjectRoot, "root", "CreateOutputFileArtifact"); var sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir); var inputB = CreateSourceFile(); var outputA = CreateOutputFileArtifact(sharedOpaqueDir); // PipA - produces a dynamic file under /root/CreateOutputFileArtifact var builderA = CreatePipBuilder(new Operation[] { Operation.WriteFile(outputA, doNotInfer: true) }); builderA.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath), SealDirectoryKind.SharedOpaque); builderA.AddTags(Context.StringTable, "pipA"); var pipA = SchedulePipBuilder(builderA); // composite shared opaque - consists of a single shared opaque produced by PipA var success = PipConstructionHelper.TryComposeSharedOpaqueDirectory(rootPath, new[] { pipA.ProcessOutputs.GetOpaqueDirectory(sharedOpaqueDirPath) }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: null, description: null, tags: new string[] { }, out var composedOpaque); XAssert.IsTrue(success); // PipB - consumes composite shared opaque directory // note: there is no direct dependency on PipA var builderB = CreatePipBuilder(new Operation[] { Operation.ReadRequiredFile(outputA, doNotInfer: true), // dynamic output of PipA Operation.ReadFile(inputB), // dummy input, so we can force pipB to re-run Operation.WriteFile(CreateOutputFileArtifact()) }); builderB.AddInputDirectory(composedOpaque); builderB.AddTags(Context.StringTable, "pipB"); var pipB = SchedulePipBuilder(builderB); // bring content into the cache RunScheduler().AssertCacheMiss(pipA.Process.PipId, pipB.Process.PipId); // Lazy materialization and tags are used here to prevent materialization of opaque directory produced by pipA, // so the only place where 'outputA' might come from is materialization of 'composedOpaque'. Configuration.Schedule.EnableLazyOutputMaterialization = true; Configuration.Filter = "tag='pipB'"; // make sure that we start with no files materialized Directory.Delete(root, recursive: true); // force pipB to re-run File.AppendAllText(ArtifactToString(inputB), "foo"); RunScheduler().AssertSuccess().AssertCacheHit(pipA.Process.PipId).AssertCacheMiss(pipB.Process.PipId); }
/// <summary> /// Pip consumes a filtered SOD (fileA, fileB) and reads both files /// Filter is changed -> SOD (fileA) /// Pip should be a miss and should emit DFA /// </summary> public void FilterChangesShouldAffectCachingIfPipReadsFilteredFile() { var includeKind = global::BuildXL.Pips.Operations.SealDirectoryContentFilter.ContentFilterKind.Include; var sharedOpaqueDir = Path.Combine(ObjectRoot, "sod"); var sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir); var sharedOpaqueDirArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath); var fileA = CreateOutputFileArtifact(sharedOpaqueDir); var fileB = CreateOutputFileArtifact(sharedOpaqueDir); var builderA = CreatePipBuilder(new Operation[] { Operation.WriteFile(fileA, doNotInfer: true), Operation.WriteFile(fileB, doNotInfer: true) }); builderA.AddOutputDirectory(sharedOpaqueDirArtifact, SealDirectoryKind.SharedOpaque); var pipA = SchedulePipBuilder(builderA); var producedSod = pipA.ProcessOutputs.GetOpaqueDirectory(sharedOpaqueDirPath); var success = PipConstructionHelper.TryComposeSharedOpaqueDirectory(sharedOpaqueDirPath, new[] { producedSod }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: new SealDirectoryContentFilter(includeKind, ".*"), description: null, tags: new string[] { }, out var filteredOpaque); XAssert.IsTrue(success); var builderB = CreateOpaqueDirectoryConsumer(CreateOutputFileArtifact(), null, filteredOpaque, fileA, fileB); var pipB = SchedulePipBuilder(builderB); // first run - populate cache RunScheduler().AssertSuccess(); // reset the graph and this time specify a filter that would exclude fileB ResetPipGraphBuilder(); pipA = SchedulePipBuilder(builderA); producedSod = pipA.ProcessOutputs.GetOpaqueDirectory(sharedOpaqueDirPath); var filter = $".*{fileA.Path.GetName(Context.PathTable).ToString(Context.StringTable)}$"; success = PipConstructionHelper.TryComposeSharedOpaqueDirectory(sharedOpaqueDirPath, new[] { producedSod }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: new SealDirectoryContentFilter(includeKind, filter), description: null, tags: new string[] { }, out filteredOpaque); XAssert.IsTrue(success); builderB = CreateOpaqueDirectoryConsumer(CreateOutputFileArtifact(), null, filteredOpaque, fileA, fileB); pipB = SchedulePipBuilder(builderB); // since filteredOpaque does not contain fileB, this run should result in a DFA IgnoreWarnings(); var result = RunScheduler().AssertFailure(); AssertErrorEventLogged(LogEventId.FileMonitoringError); result // pipA should be a hit .AssertCacheHitWithoutAssertingSuccess(pipA.Process.PipId) // pipB should be a miss (access to fileB is no longer valid) .AssertCacheMissWithoutAssertingSuccess(pipB.Process.PipId) // pipB should have a weak FP hit (file accesses are checked during strong FP computation) .AssertNumWeakFingerprintMisses(0); }
/// <summary> /// Configures the process builder to execute the specified commands /// </summary> protected virtual void ConfigureProcessBuilder( ProcessBuilder processBuilder, JavaScriptProject project) { SetCmdTool(processBuilder, project); // Working directory - the directory where the project file lives. processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder); // We allow undeclared inputs to be read processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads; // We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient // to avoid a large number of path sets processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation; // Try to preserve path set casing since many JavaScript projects deal with paths in a case-sensitive way // Otherwise in Windows we force path sets to be all uppercase processBuilder.Options |= Process.Options.PreservePathSetCasing; // By default the double write policy is to allow same content double writes. processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites; // Untrack the user profile. The corresponding mount is already configured for not tracking source files, and with allowed undeclared source reads, // any attempt to read into the user profile will fail to compute its corresponding hash processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile))); // Add the associated build script name as a tag, so filtering on 'build' or 'test' can happen processBuilder.Tags = ReadOnlyArray <StringId> .FromWithoutCopy(new[] { StringId.Create(m_context.StringTable, project.ScriptCommandName) }); PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings); var logDirectory = GetLogDirectory(project); processBuilder.SetStandardOutputFile(logDirectory.Combine(m_context.PathTable, "build.log")); processBuilder.SetStandardErrorFile(logDirectory.Combine(m_context.PathTable, "error.log")); using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " ")) { processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C")); using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, " ")) { // Execute the command and redirect the output to a designated log file processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.ScriptCommand)); // If we need to append arguments to the script command, do it here if (m_customCommands.TryGetValue(project.ScriptCommandName, out IReadOnlyList <JavaScriptArgument> extraArguments)) { foreach (JavaScriptArgument value in extraArguments) { AddJavaScriptArgumentToBuilder(processBuilder.ArgumentsBuilder, value); } } } } FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable); }
private EvaluationResult UnsafeExOutputDirectory(Context context, ModuleLiteral env, EvaluationStackFrame args) { ThrowIfNotAllowed(context, ExOutputDirectory); var path = Args.AsPath(args, 0); return(EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(path))); }
private void ProcessOutputs(ProjectWithPredictions project, ProcessBuilder processBuilder) { // Add all shared opaque directories. foreach (var sharedOutputDirectory in GetOutputDirectories(project)) { processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(sharedOutputDirectory), SealDirectoryKind.SharedOpaque); } }
public void ValidateCachingNonhashableMount() { // Absent file and directory in nonhashable mount FileArtifact nonhashableFile = new FileArtifact(CreateUniqueSourcePath(SourceRootPrefix, NonHashableRoot)); DirectoryArtifact nonhashableDir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueSourcePath(SourceRootPrefix, NonHashableRoot)); ValidateCachingBehaviorUntrackedMount(nonhashableFile, nonhashableDir, undefinedMount: false); }
/// <summary> /// Indicates that no file accesses should be tracked under the AppData directory /// </summary> /// <remarks>This includes both ApplicationData and LocalApplicationData folders</remarks> public void AddUntrackedAppDataDirectories() { if (!OperatingSystemHelper.IsUnixOS) { AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(m_pathTable, s_applicationDataPath)); AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(m_pathTable, s_localApplicationDataPath)); } }
private EvaluationResult UnsafeExOutputDirectory(Context context, ModuleLiteral env, EvaluationStackFrame args) { ThrowIfNotAllowed(context, ExOutputDirectory); var path = Args.AsPath(args, 0); return(EvaluationResult.Create(new StaticDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(path), SealDirectoryKind.Opaque, PipConstructionHelper.EmptyStaticDirContents))); }
private static EvaluationResult GetDirectoryValue(Context context, ModuleLiteral env, EvaluationStackFrame args) { AbsolutePath path = GetPathValue(context, args); return(path.IsValid ? EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(path)) : EvaluationResult.Undefined); }
public virtual void ValidateCachingUndefinedMount_Bug1087986() { // Absent file and directory in an undefined mount FileArtifact undefinedFile = new FileArtifact(CreateUniqueSourcePath(SourceRootPrefix, TemporaryDirectory)); DirectoryArtifact undefinedDir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueSourcePath(SourceRootPrefix, TemporaryDirectory)); ValidateCachingBehaviorUntrackedMount(undefinedFile, undefinedDir, undefinedMount: true); AssertWarningEventLogged(EventId.IgnoringUntrackedSourceFileNotUnderMount, 7); }
public void ValidateCachingDirectoryEnumerationReadonlyMountAllowlistFile(bool cacheableAllowlist, int mountType) { // Read-only mounts are fingerprinted based on actual filesystem state // Start with absent /allowlistFile under /dir DirectoryArtifact dir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(m_mounts[mountType])); FileArtifact allowlistFile = new FileArtifact(CreateUniquePath(SourceRootPrefix, ArtifactToString(dir))); AddAllowlistEntry(allowlistFile, cacheableAllowlist: cacheableAllowlist); FileArtifact outFile = CreateOutputFileArtifact(); Process pip = CreateAndSchedulePipBuilder(new Operation[] { Operation.EnumerateDir(dir), Operation.WriteFile(outFile) }).Process; RunScheduler().AssertCacheMiss(pip.PipId); RunScheduler().AssertCacheHit(pip.PipId); string outputWithAbsentAllowlistFile = File.ReadAllText(ArtifactToString(outFile)); // Create /allowlistFile File.WriteAllText(ArtifactToString(allowlistFile), "abc"); switch (mountType) { case MountType.Readonly: // allowlisted files still make it into directory fingerprints // This is consistent with tracking existence of allowlisted files for absent file probes RunScheduler().AssertCacheMiss(pip.PipId); break; case MountType.NonHashable: // Files in nonhashable mounts never make it into directory fingerprints break; case MountType.Undefined: // Files in undefined mounts never make it into directory fingerprints // Bug 1134297: This is inconsistent with our tracking of absent file probes for allowlisted // files under undefined mounts break; default: return; } RunScheduler().AssertCacheHit(pip.PipId); string outputWithExistingAllowlistFile = File.ReadAllText(ArtifactToString(outFile)); // Delete /allowlistFile File.Delete(ArtifactToString(allowlistFile)); RunScheduler().AssertCacheHit(pip.PipId); XAssert.AreEqual(File.ReadAllText(ArtifactToString(outFile)), outputWithAbsentAllowlistFile); // Re-create /allowlistFile File.WriteAllText(ArtifactToString(allowlistFile), "xyz"); RunScheduler().AssertCacheHit(pip.PipId); XAssert.AreEqual(File.ReadAllText(ArtifactToString(outFile)), outputWithExistingAllowlistFile); }
/// <nodoc /> public MacOsDefaults(PathTable pathTable, PipGraph.Builder pipGraph) { m_provenance = new PipProvenance( 0, ModuleId.Invalid, StringId.Invalid, FullSymbol.Invalid, LocationData.Invalid, QualifierId.Unqualified, PipData.Invalid); m_inputFiles = new[] { FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.Etc)), FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.TmpDir)), }; // Sealed Source inputs m_inputDirectories = new[] { GetSourceSeal(pathTable, pipGraph, MacPaths.Applications), GetSourceSeal(pathTable, pipGraph, MacPaths.UsrBin), GetSourceSeal(pathTable, pipGraph, MacPaths.UsrInclude), GetSourceSeal(pathTable, pipGraph, MacPaths.UsrLib), GetSourceSeal(pathTable, pipGraph, MacPaths.Library), GetSourceSeal(pathTable, pipGraph, MacPaths.UserProvisioning), }; m_untrackedFiles = new[] { // login.keychain is created by the OS the first time any process invokes an OS API that references the keychain. // Untracked because build state will not be stored there and code signing will fail if required certs are in the keychain FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserKeyChainsDb)), FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserKeyChains)), FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserCFTextEncoding)), }; m_untrackedDirectories = new[] { DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Bin), DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Dev), DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Private), DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Sbin), DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.SystemLibrary), DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrLibexec), DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrShare), DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrStandalone), DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrSbin), DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Var), DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UserPreferences), }; }
/// <inheritdoc /> public DirectoryArtifact AddSealDirectory(SealDirectory sealDirectory, PipId valuePip) { Contract.Requires(sealDirectory != null); sealDirectory.PipId = GetPipId(); sealDirectory.SetDirectoryArtifact(DirectoryArtifact.CreateWithZeroPartialSealId(sealDirectory.DirectoryRoot)); Contract.Assume(sealDirectory.IsInitialized); m_pips.Enqueue(sealDirectory); return(sealDirectory.Directory); }
private static EvaluationResult GetUserHomeDirectory(Context context, ModuleLiteral env, EvaluationStackFrame args) { string homePath = Host.Current.CurrentOS == BuildXL.Interop.OperatingSystem.Win ? I($"{Environment.GetEnvironmentVariable(")HOMEDRIVE") ?? "C:"}{Path.DirectorySeparatorChar}{Environment.GetEnvironmentVariable("HOMEPATH") ?? "Users"}") : Environment.GetEnvironmentVariable("HOME"); var directoryPath = AbsolutePath.Create(context.FrontEndContext.PathTable, homePath); return(EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(directoryPath))); }
private DirectoryArtifact Convert(DirectoryArtifact directory) { if (AreGraphsSame) { return(directory); } // TODO: This is wrong (loses the seal id but it probably doesn't matter since this isn't used for comparison) return(DirectoryArtifact.CreateWithZeroPartialSealId(Convert(directory.Path))); }
/// <summary> /// Dumps file members into an array of file artifacts. /// </summary> public void GetArtifacts(Context context, EvaluationResult[] files, bool asDirectory) { for (int i = 0; i < m_files.Count; ++i) { var path = DirectoryPath.Combine(context.PathTable, m_files[i]); files[Index + i] = asDirectory ? EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(path)) : EvaluationResult.Create(FileArtifact.CreateSourceFile(path)); } }