private Task <SandboxedProcessResult> RunJavaScriptGraphBuilderAsync(
string nodeExeLocation,
AbsolutePath outputFile,
BuildParameters.IBuildParameters buildParameters,
AbsolutePath toolLocation)
{
AbsolutePath toolPath = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool);
string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable);
var cmdExeArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(m_context.PathTable, Environment.GetEnvironmentVariable("COMSPEC")));
var toolArguments = GetGraphConstructionToolArguments(outputFile, toolLocation, toolPath, nodeExeLocation);
Tracing.Logger.Log.ConstructingGraphScript(m_context.LoggingContext, toolArguments);
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
cmdExeArtifact.Path.ToString(m_context.PathTable),
buildStorageDirectory: outputDirectory,
fileAccessManifest: FrontEndUtilities.GenerateToolFileAccessManifest(m_context, outputFile.GetParent(m_context.PathTable)),
arguments: toolArguments,
workingDirectory: m_resolverSettings.Root.ToString(m_context.PathTable),
description: $"{Name} graph builder",
buildParameters));
}
public void CreateFileArtifact()
{
var pt = new PathTable();
AbsolutePath da1 = AbsolutePath.Create(pt, A("c"));
RelativePath relPath = RelativePath.Create(pt.StringTable, R("a", "b"));
FileArtifact fa = FileArtifact.CreateSourceFile(da1.Combine(pt, relPath));
XAssert.AreEqual(A("c", "a", "b"), fa.Path.ToString(pt));
da1 = AbsolutePath.Create(pt, A("c"));
PathAtom atom = PathAtom.Create(pt.StringTable, "a");
fa = FileArtifact.CreateSourceFile(da1.Combine(pt, atom));
XAssert.AreEqual(A("c", "a"), fa.Path.ToString(pt));
da1 = AbsolutePath.Create(pt, A("c"));
atom = PathAtom.Create(pt.StringTable, "a");
PathAtom atom2 = PathAtom.Create(pt.StringTable, "b");
fa = FileArtifact.CreateSourceFile(da1.Combine(pt, atom, atom2));
XAssert.AreEqual(A("c", "a", "b"), fa.Path.ToString(pt));
da1 = AbsolutePath.Create(pt, A("c"));
atom = PathAtom.Create(pt.StringTable, "a");
atom2 = PathAtom.Create(pt.StringTable, "b");
PathAtom atom3 = PathAtom.Create(pt.StringTable, "c");
fa = FileArtifact.CreateSourceFile(da1.Combine(pt, atom, atom2, atom3));
XAssert.AreEqual(A("c", "a", "b", "c"), fa.Path.ToString(pt));
}
public void TestPipOutputFingerprinting(FileExistence existence, FileExistence anotherExistence)
{
// Arrange
var pathTable = m_context.PathTable;
var executable = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, X("/x/pkgs/tool.exe")));
AbsolutePath outputPath = AbsolutePath.Create(pathTable, X("/x/obj/working/out.bin"));
FileArtifactWithAttributes output = new FileArtifactWithAttributes(outputPath, rewriteCount: 1, fileExistence: existence);
Process process = DefaultBuilder.WithOutputs(output).Build();
Process processSecondVersion = DefaultBuilder.WithOutputs(output.CreateNextWrittenVersion()).Build();
var outputWithDifferenceExistence = new FileArtifactWithAttributes(outputPath, rewriteCount: 1, fileExistence: anotherExistence);
var processWithDifferenceExistence = DefaultBuilder.WithOutputs(outputWithDifferenceExistence).Build();
var fingerprinter = CreateFingerprinter(executable);
// Act
var fingerprint = fingerprinter.ComputeFingerprint(process);
// Assert
XAssert.AreEqual(fingerprint, fingerprinter.ComputeFingerprint(processSecondVersion),
"RewriteCount should not affect the fingerprint");
XAssert.AreNotEqual(fingerprint, fingerprinter.ComputeFingerprint(processWithDifferenceExistence),
"Two process with the same output path but with different attributes should produce different fingerprints");
}
private void AddInputs(NinjaNode node, ProcessBuilder processBuilder)
{
foreach (AbsolutePath input in node.Inputs.Where(i => !m_outputFileArtifacts.ContainsKey(i))) // We only want source files. If the file is in m_outputFileArtifacts then it was output by someone else
{ // and we have that dependency expressed in node.
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(input));
}
foreach (NinjaNode dependency in node.Dependencies)
{
bool outputsPresent = m_processOutputs.TryGetValue(dependency, out ProcessOutputs processOutputs);
string ListOutputs(NinjaNode n) => string.Join(" ", n.Outputs.Select(x => x.GetName(m_context.PathTable).ToString(m_context.StringTable)).ToList());
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: [build { ListOutputs(dependency) }] missing, dependency of [build { ListOutputs(node)} ]");
}
foreach (FileArtifact output in processOutputs.GetOutputFiles())
{
if (node.Inputs.Contains(output.Path))
{
processBuilder.AddInputFile(output);
}
}
}
}
/// <nodoc />
public bool TryWriteFile(
AbsolutePath destination,
PipData content,
WriteFileEncoding encoding,
string[] tags,
string description,
out FileArtifact fileArtifact)
{
Contract.Requires(destination.IsValid);
Contract.Requires(content.IsValid);
fileArtifact = FileArtifact.CreateSourceFile(destination).CreateNextWrittenVersion();
var pip = new WriteFile(
fileArtifact,
content,
encoding,
ToStringIds(tags),
CreatePipProvenance(description));
if (PipGraph != null)
{
return(PipGraph.AddWriteFile(pip, GetValuePipId()));
}
return(true);
}
/// <summary>
/// Creates a source artifact and populates it with <see cref="WriteSourceFile"/>.
/// Creating a backing file is necessary since source artifacts must exist at the beginning of a build.
///
/// If <paramref name="root"/> is not specified, <see cref="SourceRoot"/> is used as the parent directory.
///
/// If <paramref name="prefix"/> is not specified, <see cref="SourceRootPrefix"/> is used as the file name prefix.
/// </summary>
protected FileArtifact CreateSourceFileWithPrefix(string root = null, string prefix = null)
{
FileArtifact sourceFile = FileArtifact.CreateSourceFile(CreateUniqueSourcePath(prefix ?? SourceRootPrefix, root));
WriteSourceFile(sourceFile);
return(sourceFile);
}
protected PipTestBase(ITestOutputHelper output) : base(output)
{
Context = BuildXLContext.CreateInstanceForTesting();
PathTable.DebugPathTable = Context.PathTable;
TestBinRoot = Path.GetDirectoryName(AssemblyHelper.GetAssemblyLocation(System.Reflection.Assembly.GetExecutingAssembly()));
TestBinRootPath = AbsolutePath.Create(Context.PathTable, TestBinRoot);
SourceRoot = Path.Combine(TemporaryDirectory, SourceRootPrefix);
SourceRootPath = AbsolutePath.Create(Context.PathTable, SourceRoot);
CmdExecutable = FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, CmdHelper.OsShellExe));
string testProcessFolder = Path.Combine(TestBinRoot, "TestProcess");
string platformDir = Dispatch.CurrentOS().ToString();
string exe = Path.Combine(testProcessFolder, platformDir, TestProcessToolName);
TestProcessExecutable = FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, exe));
// Test process depends on C://Windows (when running on Windows)
TestProcessDependencies = OperatingSystemHelper.IsUnixOS
? new AbsolutePath[0]
: new AbsolutePath[] { AbsolutePath.Create(Context.PathTable, Environment.GetFolderPath(Environment.SpecialFolder.Windows)) };
ObjectRoot = Path.Combine(TemporaryDirectory, ObjectRootPrefix);
ObjectRootPath = AbsolutePath.Create(Context.PathTable, ObjectRoot);
CacheRoot = Path.Combine(TemporaryDirectory, CacheRootPrefix);
BaseSetup();
}
/// <nodoc />
public bool TryCopyFile(
FileArtifact source,
AbsolutePath destination,
CopyFile.Options options,
string[] tags,
string description,
out FileArtifact fileArtifact)
{
Contract.Requires(source.IsValid);
Contract.Requires(destination.IsValid);
fileArtifact = FileArtifact.CreateSourceFile(destination).CreateNextWrittenVersion();
var pip = new CopyFile(
source,
fileArtifact,
ToStringIds(tags),
CreatePipProvenance(description),
options);
if (PipGraph != null)
{
return(PipGraph.AddCopyFile(pip, GetValuePipId()));
}
return(true);
}
/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
ProjectWithPredictions project,
ProcessBuilder processBuilder)
{
// Add all predicted inputs
foreach (AbsolutePath buildInput in project.PredictedInputFiles)
{
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput));
}
IEnumerable <ProjectWithPredictions> references;
// The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in
if (m_resolverSettings.EnableTransitiveProjectReferences == true)
{
// In this case all the transitive closure is automatically exposed to the project as direct references
var transitiveReferences = new HashSet <ProjectWithPredictions>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
references = transitiveReferences;
}
else
{
// Only direct dependencies are declared.
// Add all known explicit inputs from project references. But rule out
// projects that have a known empty list of targets: those projects are not scheduled, so
// there is nothing to consume from them.
references = project.ProjectReferences.Where(projectReference => !projectReference.PredictedTargetsToExecute.TargetsAreKnownToBeEmpty);
}
var argumentsBuilder = processBuilder.ArgumentsBuilder;
foreach (ProjectWithPredictions projectReference in references)
{
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out MSBuildProjectOutputs projectOutputs);
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}");
}
// Add all known output directories
foreach (StaticDirectory output in projectOutputs.OutputDirectories)
{
processBuilder.AddInputDirectory(output.Root);
}
// If the dependency was built in isolation, this project needs to access the generated cache files
if (projectOutputs.BuildsInIsolation)
{
var outputCache = projectOutputs.OutputCacheFile;
processBuilder.AddInputFile(outputCache);
// Instruct MSBuild to use the cache file from the associated dependency as an input.
// Flag /irc is the short form of /inputResultsCaches, and part of MSBuild 'build in isolation' mode.
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
argumentsBuilder.Add(PipDataAtom.FromString("/irc:"));
argumentsBuilder.Add(PipDataAtom.FromAbsolutePath(outputCache));
}
}
}
}
private Task <SandboxedProcessResult> RunRushGraphBuilderAsync(
AbsolutePath outputFile,
BuildParameters.IBuildParameters buildParameters)
{
AbsolutePath toolPath = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool);
string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable);
// We always use cmd.exe as the tool so if the node.exe location is not provided we can just pass 'node.exe' and let PATH do the work.
var cmdExeArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(m_context.PathTable, Environment.GetEnvironmentVariable("COMSPEC")));
string nodeExe = m_resolverSettings.NodeExeLocation.HasValue ?
m_resolverSettings.NodeExeLocation.Value.Path.ToString(m_context.PathTable) :
"node.exe";
string pathToRushJson = m_resolverSettings.Root.Combine(m_context.PathTable, "rush.json").ToString(m_context.PathTable);
// TODO: add qualifier support.
// The graph construction tool expects: <path-to-rush.json> <path-to-output-graph> [<debug|release>]
string toolArguments = $@"/C """"{nodeExe}"" ""{toolPath.ToString(m_context.PathTable)}"" ""{pathToRushJson}"" ""{outputFile.ToString(m_context.PathTable)}"" debug""";
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
cmdExeArtifact.Path.ToString(m_context.PathTable),
buildStorageDirectory: outputDirectory,
fileAccessManifest: FrontEndUtilities.GenerateToolFileAccessManifest(m_context, outputFile.GetParent(m_context.PathTable)),
arguments: toolArguments,
workingDirectory: m_configuration.Layout.SourceDirectory.ToString(m_context.PathTable),
description: "Rush graph builder",
buildParameters));
}
/// <nodoc />
private async Task <EvaluationResult> ValidateAndStoreIncrementalDownloadStateAsync(DownloadData downloadData)
{
// If the hash is given in the download setting, use the corresponding hashType(hash algorithm) to get the content hash of the downloaded file.
var downloadedHash = await GetContentHashAsync(downloadData.DownloadedFilePath, HashTypeParser(downloadData.Settings.Hash));
if (downloadData.ContentHash.HasValue)
{
// Validate downloaded hash if specified
if (downloadData.ContentHash != downloadedHash)
{
m_logger.DownloadMismatchedHash(
m_context.LoggingContext,
downloadData.Settings.ModuleName,
downloadData.Settings.Url,
downloadData.Settings.Hash,
downloadedHash.ToString());
return(EvaluationResult.Error);
}
}
else
{
try
{
var incrementalState = new DownloadIncrementalState(downloadData, downloadedHash);
await incrementalState.SaveAsync(m_context);
}
catch (BuildXLException e)
{
m_logger.ErrorStoringIncrementality(m_context.LoggingContext, downloadData.Settings.ModuleName, e.Message);
return(EvaluationResult.Error);
}
}
return(new EvaluationResult(FileArtifact.CreateSourceFile(downloadData.DownloadedFilePath)));
}
private bool TrySetBuildToolExecutor(
PipConstructionHelper pipConstructionHelper,
ProcessBuilder processBuilder,
ProjectWithPredictions project)
{
// If we should use the dotnet core version of msbuild, the executable for the pip is dotnet.exe instead of msbuild.exe, and
// the first argument is msbuild.dll
FileArtifact cmdExeArtifact;
if (m_resolverSettings.ShouldRunDotNetCoreMSBuild())
{
cmdExeArtifact = FileArtifact.CreateSourceFile(m_dotnetExePath);
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(m_msBuildPath));
}
else
{
cmdExeArtifact = FileArtifact.CreateSourceFile(m_msBuildPath);
}
processBuilder.Executable = cmdExeArtifact;
processBuilder.AddInputFile(cmdExeArtifact);
processBuilder.AddCurrentHostOSDirectories();
processBuilder.AddUntrackedAppDataDirectories();
processBuilder.AddUntrackedProgramDataDirectories();
// Temp directory setup including setting TMP and TEMP env vars. The path to
// the temp dir is generated in a consistent fashion between BuildXL runs to
// ensure environment value (and hence pip hash) consistency.
processBuilder.EnableTempDirectory();
processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {project.FullPath.ToString(PathTable)}"));
return(true);
}
public void ProbeWithinSealedSourceDirectory()
{
var output = CreateOutputFileArtifact(ObjectRoot);
var input = CreateSourceFile(SourceRoot);
var absentPath = FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRoot));
DirectoryArtifact dir = SealDirectory(SourceRootPath, SealDirectoryKind.SourceAllDirectories);
var builder = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(input),
Operation.ReadFile(absentPath, doNotInfer: true),
Operation.WriteFile(output)
});
builder.AddInputDirectory(dir);
Process p = SchedulePipBuilder(builder).Process;
RunScheduler().AssertSuccess();
RunScheduler().AssertCacheHit(p.PipId);
// Make the absent path exist and make sure it is cached appropriately
WriteSourceFile(absentPath);
RunScheduler().AssertFailure();
AssertVerboseEventLogged(ProcessesLogEventId.PipProcessDisallowedFileAccess);
AssertVerboseEventLogged(LogEventId.DependencyViolationMissingSourceDependency);
AssertWarningEventLogged(LogEventId.ProcessNotStoredToCacheDueToFileMonitoringViolations);
AssertErrorEventLogged(LogEventId.FileMonitoringError);
}
public void ConsumeFilterPassingFile(bool topOnly)
{
FileArtifact source;
if (topOnly)
{
source = FileArtifact.CreateSourceFile(SourceRootPath.Combine(Context.PathTable, "file.txt"));
}
else
{
var nestedDir = AbsolutePath.Create(Context.PathTable, Path.Combine(SourceRoot, "nested"));
source = FileArtifact.CreateSourceFile(nestedDir.Combine(Context.PathTable, "file.txt"));
}
WriteSourceFile(source);
var output = CreateOutputFileArtifact(ObjectRoot);
SealDirectory sealedDirectory = CreateSourceSealDirectory(SourceRootPath, topOnly ? SealDirectoryKind.SourceTopDirectoryOnly : SealDirectoryKind.SourceAllDirectories, "*.txt", "*.cs");
DirectoryArtifact dir = PipGraphBuilder.AddSealDirectory(sealedDirectory);
var builder = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(source, doNotInfer: true),
Operation.WriteFile(output),
});
builder.AddInputDirectory(dir);
SchedulePipBuilder(builder);
RunScheduler().AssertSuccess();
}
private static void ScheduleConsumeSourceFile(TestEnv env, AbsolutePath directory, string relativePath)
{
FileArtifact source = FileArtifact.CreateSourceFile(env.Paths.CreateAbsolutePath(directory, env.Paths.CreateRelativePath(relativePath)));
AbsolutePath output = env.Paths.CreateAbsolutePath(env.ObjectRoot, Guid.NewGuid().ToString());
env.PipConstructionHelper.TryCopyFile(source, output, CopyFile.Options.None, null, "Pretend a source file is used", out _);
}
public void ConsumeFilterNotPassingFile()
{
FileArtifact source = FileArtifact.CreateSourceFile(SourceRootPath.Combine(Context.PathTable, "file.txt"));
WriteSourceFile(source);
var output = CreateOutputFileArtifact(ObjectRoot);
SealDirectory sealedDirectory = CreateSourceSealDirectory(SourceRootPath, SealDirectoryKind.SourceTopDirectoryOnly, "*.cs");
DirectoryArtifact dir = PipGraphBuilder.AddSealDirectory(sealedDirectory);
var builder = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(source, doNotInfer: true),
Operation.WriteFile(output),
});
builder.AddInputDirectory(dir);
SchedulePipBuilder(builder);
RunScheduler().AssertFailure();
AssertVerboseEventLogged(ProcessesLogEventId.PipProcessDisallowedFileAccess);
AssertVerboseEventLogged(LogEventId.DependencyViolationMissingSourceDependency);
AssertWarningEventLogged(LogEventId.ProcessNotStoredToCacheDueToFileMonitoringViolations);
AssertErrorEventLogged(LogEventId.FileMonitoringError);
}
public void TestFailingPipDump(bool failPip)
{
BaseSetup();
FileArtifact sourceArtifact = failPip ? FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)) : CreateSourceFile();
FileArtifact destinationArtifact = CreateOutputFileArtifact();
var copyFile = CreateCopyFile(sourceArtifact, destinationArtifact);
PipGraphBuilder.AddCopyFile(copyFile);
var schedulerResult = RunScheduler();
var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips");
var pipDumpFile = Path.Combine(logFolder, $"{copyFile.FormattedSemiStableHash}.json");
if (failPip)
{
schedulerResult.AssertFailure();
SetExpectedFailures(1, 0); // One error logged for the failing pip
AssertErrorCount();
Assert.True(Directory.Exists(logFolder));
Assert.True(File.Exists(pipDumpFile));
Assert.True(new FileInfo(pipDumpFile).Length > 0); //Ensure that some content was written to the file
}
else
{
schedulerResult.AssertSuccess();
// FailedPips log folder should not have been created when we run successfully
Assert.False(Directory.Exists(logFolder));
Assert.False(File.Exists(pipDumpFile));
}
}
private void DoTestRenderer(PathTable pathTable, PipFragmentRenderer renderer, string expectedHash)
{
// StringId
var strValue = "my string";
XAssert.AreEqual(strValue, renderer.Render(PipFragment.FromString(strValue, pathTable.StringTable)));
var pathStr = A("t", "file1.txt");
var path = AbsolutePath.Create(pathTable, pathStr);
var srcFile = FileArtifact.CreateSourceFile(path);
var outFile = FileArtifact.CreateOutputFile(srcFile);
var rwFile = outFile.CreateNextWrittenVersion();
var rw2File = rwFile.CreateNextWrittenVersion();
var opaqueDir = DirectoryArtifact.CreateDirectoryArtifactForTesting(path, 0);
var sharedDir = new DirectoryArtifact(path, 1, isSharedOpaque: true);
// AbsolutePath
XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(path)));
XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(srcFile)));
XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(outFile)));
XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(rwFile)));
XAssert.AreEqual(pathStr, renderer.Render(PipFragment.FromAbsolutePathForTesting(rw2File)));
// VsoHash
XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(srcFile)));
XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(outFile)));
XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(rwFile)));
XAssert.AreEqual(expectedHash, renderer.Render(PipFragment.VsoHashFromFileForTesting(rw2File)));
XAssert.AreEqual(DirectoryId.ToString(opaqueDir), renderer.Render(PipFragment.DirectoryIdForTesting(opaqueDir)));
XAssert.AreEqual(DirectoryId.ToString(sharedDir), renderer.Render(PipFragment.DirectoryIdForTesting(sharedDir)));
}
public void TestDumpPipLiteWithPassingAndFailingPips()
{
var passingCopyFile = new CopyFile(CreateSourceFile(), CreateOutputFileArtifact(), new List <StringId>().ToReadOnlyArray(), PipProvenance.CreateDummy(Context));
var failingCopyFile = new CopyFile(FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)), CreateOutputFileArtifact(), new List <StringId>().ToReadOnlyArray(), PipProvenance.CreateDummy(Context));
PipGraphBuilder.AddCopyFile(passingCopyFile);
PipGraphBuilder.AddCopyFile(failingCopyFile);
var schedulerResult = RunScheduler().AssertFailure();
var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips");
var failingPipDumpFile = Path.Combine(logFolder, $"{failingCopyFile.FormattedSemiStableHash}.json");
var passingPipDumpFile = Path.Combine(logFolder, $"{passingCopyFile.FormattedSemiStableHash}.json");
List <Option> options = new List <Option>
{
new Option()
{
Name = "d+"
}
};
RunAnalyzer(schedulerResult, null, options);
Assert.True(Directory.Exists(logFolder));
Assert.True(File.Exists(failingPipDumpFile));
Assert.True(!File.Exists(passingPipDumpFile));
}
public void AffectedSourceInsideSourceSealDirectory()
{
var sourceDirectory = CreateUniqueDirectory(SourceRootPath);
var fileInsideSourceDirectory = CreateSourceFile(sourceDirectory, "file_");
var sourceSealDirectory = CreateSourceSealDirectory(sourceDirectory, SealDirectoryKind.SourceTopDirectoryOnly, "file_*");
PipGraphBuilder.AddSealDirectory(sourceSealDirectory);
var changeAffectedWrittenFile = CreateUniqueObjPath("change");
var pipBuilder = CreatePipBuilder(new Operation[]
{
// Ensure that pip reads the changeAffectedWrittenFile if it exists.
Operation.ReadFile(FileArtifact.CreateSourceFile(changeAffectedWrittenFile), doNotInfer: true),
Operation.ReadFile(fileInsideSourceDirectory, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()),
});
pipBuilder.AddInputDirectory(sourceSealDirectory.Directory);
pipBuilder.SetChangeAffectedInputListWrittenFile(changeAffectedWrittenFile);
SchedulePipBuilder(pipBuilder);
var inputChangesFile = CreateOutputFileArtifact();
File.WriteAllText(ArtifactToString(inputChangesFile), ArtifactToString(fileInsideSourceDirectory));
Configuration.Schedule.InputChanges = inputChangesFile;
RunScheduler().AssertSuccess();
var actualAffectedInput = File.ReadAllText(changeAffectedWrittenFile.ToString(Context.PathTable));
XAssert.AreEqual(ArtifactToString(fileInsideSourceDirectory), actualAffectedInput);
}
private ContentFingerprint CreateFingerprintForPartialSealWithMember(string fileName)
{
using (TestEnv env = TestEnv.CreateTestEnvWithPausedScheduler())
{
AbsolutePath root = env.Paths.CreateAbsolutePath(@"\\dummyPath\Root");
FileArtifact member = FileArtifact.CreateSourceFile(root.Combine(env.PathTable, fileName));
var staticDirectory = env.PipConstructionHelper.SealDirectoryPartial(
root,
new[] { member });
var pipBuilder = CreatePipBuilderWithTag(env, nameof(TestPartialSealDirectoryMembersAffectFingerprints));
var outputPath = env.Paths.CreateAbsolutePath(@"\\dummyPath\out");
pipBuilder.AddOutputFile(outputPath);
pipBuilder.AddInputDirectory(staticDirectory);
env.PipConstructionHelper.AddProcess(pipBuilder);
var graph = AssertSuccessGraphBuilding(env);
var producerId = graph.TryGetProducer(FileArtifact.CreateOutputFile(outputPath));
XAssert.IsTrue(producerId.IsValid);
XAssert.IsTrue(graph.TryGetPipFingerprint(producerId, out ContentFingerprint fingerprint));
return(fingerprint);
}
}
public void IncrementalSchedulingIsRobustAgainstFileCreationMidBuild()
{
var directoryPath = CreateUniqueDirectory(ReadonlyRoot);
var fileInsideDirectory = FileArtifact.CreateSourceFile(Combine(directoryPath, "fileToBeCreated"));
FileArtifact outputA = CreateOutputFileArtifact();
SealDirectory sealedDirectory = CreateAndScheduleSealDirectory(directoryPath, SealDirectoryKind.Partial, fileInsideDirectory);
var builderA = CreatePipBuilder(new Operation[]
{
Operation.Probe(fileInsideDirectory, doNotInfer: true),
Operation.WriteFile(outputA, "Hello World A!")
});
builderA.AddInputDirectory(sealedDirectory.Directory);
var pipA = SchedulePipBuilder(builderA);
FileArtifact outputB = CreateOutputFileArtifact();
FileArtifact rougeOutput = CreateOutputFileArtifact(directoryPath);
var builderB = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(outputA),
Operation.WriteFile(outputB, "Hello World B!"),
Operation.WriteFile(fileInsideDirectory, "Whatever", doNotInfer: true)
});
builderB.AddUntrackedDirectoryScope(sealedDirectory.Directory);
var pipB = SchedulePipBuilder(builderB);
FileArtifact outputC = CreateOutputFileArtifact();
var builderC = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(outputB),
Operation.Probe(fileInsideDirectory, doNotInfer: true),
Operation.WriteFile(outputC, "Hello World C!")
});
builderC.AddInputDirectory(sealedDirectory.Directory);
var pipC = SchedulePipBuilder(builderC);
var result = RunScheduler().AssertScheduled(
pipA.Process.PipId,
pipB.Process.PipId,
pipC.Process.PipId);
// File creation blows away incremental scheduling state.
RunScheduler()
.AssertScheduled(
pipA.Process.PipId,
pipB.Process.PipId,
pipC.Process.PipId)
.AssertCacheMiss(pipA.Process.PipId, pipC.Process.PipId)
.AssertCacheHit(pipB.Process.PipId);
}
public void AllowlistOnSpawnProcess(bool includeExecutableLink)
{
FileArtifact exe = FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, CmdHelper.OsShellExe));
FileArtifact exeLink = FileArtifact.CreateSourceFile(CreateUniqueSourcePath());
XAssert.IsTrue(FileUtilities.TryCreateSymbolicLink(exeLink.Path.ToString(Context.PathTable), exe.Path.ToString(Context.PathTable), true).Succeeded);
Configuration.CacheableFileAccessAllowlist.Add(new Configuration.Mutable.FileAccessAllowlistEntry()
{
ToolPath = exeLink, PathRegex = ".*"
});
FileArtifact output = CreateOutputFileArtifact();
var builder = CreatePipBuilder(new[]
{
Operation.SpawnExe(
Context.PathTable,
exeLink,
string.Format(OperatingSystemHelper.IsUnixOS ? "-c \"echo 'hi' > {0}\"" : "/d /c echo 'hi' > {0}", output.Path.ToString(Context.PathTable))),
});
builder.AddOutputFile(output.Path);
if (includeExecutableLink)
{
builder.AddInputFile(exeLink);
}
foreach (var dep in CmdHelper.GetCmdDependencies(Context.PathTable))
{
builder.AddUntrackedFile(dep);
}
foreach (var dep in CmdHelper.GetCmdDependencyScopes(Context.PathTable))
{
builder.AddUntrackedDirectoryScope(dep);
}
SchedulePipBuilder(builder);
if (includeExecutableLink)
{
RunScheduler().AssertSuccess();
XAssert.AreEqual("hi", File.ReadAllText(ArtifactToString(output)).Trim().Trim('\''));
}
else
{
RunScheduler().AssertFailure();
// DFA on exeLink because it is not specified as input.
// Although there's a cacheable allowlist entry for exeLink, that entry only holds for file accessed by exeLink.
// In this case, exeLink is accessed by the test process, so there's a read operation by the test process on exeLink, hence DFA.
AssertErrorEventLogged(LogEventId.FileMonitoringError, 1);
AssertLogContains(false, $"R {exeLink.Path.ToString(Context.PathTable)}");
AssertWarningEventLogged(LogEventId.ProcessNotStoredToCacheDueToFileMonitoringViolations, 1);
}
}
private static EvaluationResult GetFileValue(Context context, ModuleLiteral env, EvaluationStackFrame args)
{
AbsolutePath path = GetPathValue(context, args);
return(path.IsValid
? EvaluationResult.Create(FileArtifact.CreateSourceFile(path))
: EvaluationResult.Undefined);
}
/// <summary>
/// Creates a symbolic link to a shell executable and returns a FileArtifact with a path to the executable
/// </summary>
protected FileArtifact CreateOsShellExecutableSymbolicLink()
{
FileArtifact exe = FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, CmdHelper.OsShellExe));
FileArtifact exeLink = FileArtifact.CreateSourceFile(CreateUniqueSourcePath());
XAssert.IsTrue(FileUtilities.TryCreateSymbolicLink(exeLink.Path.ToString(Context.PathTable), exe.Path.ToString(Context.PathTable), true).Succeeded);
return(exeLink);
}
public Task DirectlyReportedNonexistentFailsWithAllowRead(AccessType accessType)
{
var file = FileArtifact.CreateSourceFile(CreateUniqueSourcePath());
return(VerifyReporting(
accessType,
(manifest) => manifest.AddPath(file, values: FileAccessPolicy.AllowRead | FileAccessPolicy.ReportAccess, mask: FileAccessPolicy.MaskNothing),
ExpectDeniedAccess(file, exists: false)));
}
/// <nodoc />
public static FileArtifact GetOutputFile(this ProcessOutputs processOutputs, AbsolutePath path)
{
if (!processOutputs.TryGetOutputFile(FileArtifact.CreateSourceFile(path), out var file))
{
throw new InvalidOperationException();
}
return(file);
}
/// <summary>
/// Gets a process builder.
/// </summary>
public ProcessBuilder GetProcessBuilder()
{
var builder = ProcessBuilder.CreateForTesting(Context.PathTable);
builder.Executable = FileArtifact.CreateSourceFile(m_sourceRoot.Combine(Context.PathTable, "test.exe"));
builder.AddInputFile(builder.Executable);
return(builder);
}
/// <nodoc />
public MacOsDefaults(PathTable pathTable, PipGraph.Builder pipGraph)
{
m_provenance = new PipProvenance(
0,
ModuleId.Invalid,
StringId.Invalid,
FullSymbol.Invalid,
LocationData.Invalid,
QualifierId.Unqualified,
PipData.Invalid);
m_inputFiles =
new[]
{
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.Etc)),
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.TmpDir)),
};
// Sealed Source inputs
m_inputDirectories =
new[]
{
GetSourceSeal(pathTable, pipGraph, MacPaths.Applications),
GetSourceSeal(pathTable, pipGraph, MacPaths.UsrBin),
GetSourceSeal(pathTable, pipGraph, MacPaths.UsrInclude),
GetSourceSeal(pathTable, pipGraph, MacPaths.UsrLib),
GetSourceSeal(pathTable, pipGraph, MacPaths.Library),
GetSourceSeal(pathTable, pipGraph, MacPaths.UserProvisioning),
};
m_untrackedFiles =
new[]
{
// login.keychain is created by the OS the first time any process invokes an OS API that references the keychain.
// Untracked because build state will not be stored there and code signing will fail if required certs are in the keychain
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserKeyChainsDb)),
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserKeyChains)),
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserCFTextEncoding)),
};
m_untrackedDirectories =
new[]
{
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Bin),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Dev),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Private),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Sbin),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.SystemLibrary),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrLibexec),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrShare),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrStandalone),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrSbin),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Var),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UserPreferences),
};
}
/// <nodoc />
private async Task <EvaluationResult> ValidateAndStoreIncrementalExtractState(PipConstructionHelper pipConstructionHelper, DownloadData downloadData)
{
var archive = downloadData.DownloadedFilePath.ToString(m_context.PathTable);
var target = downloadData.ContentsFolder.Path.ToString(m_context.PathTable);
try
{
var allFiles = new List <FileArtifact>();
var enumeratResult = FileUtilities.EnumerateDirectoryEntries(target, true, "*",
(dir, fileName, attributes) =>
{
if ((attributes & FileAttributes.Directory) == 0)
{
var filePath = Path.Combine(dir, fileName);
allFiles.Add(FileArtifact.CreateSourceFile(AbsolutePath.Create(m_context.PathTable, filePath)));
}
});
if (!enumeratResult.Succeeded)
{
var error = new Win32Exception(enumeratResult.NativeErrorCode);
m_logger.ErrorListingPackageContents(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, error.Message);
return(EvaluationResult.Error);
}
if (allFiles.Count == 0)
{
m_logger.ErrorListingPackageContents(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, "file list is empty");
return(EvaluationResult.Error);
}
var sortedFiles = SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(
allFiles,
OrdinalFileArtifactComparer.Instance);
var hashes = new Dictionary <AbsolutePath, ContentHash>();
foreach (var file in allFiles)
{
var hash = await GetContentHashAsync(file);
hashes.Add(file.Path, hash);
}
var incrementalState = new ExtractIncrementalState(downloadData, sortedFiles, hashes);
await incrementalState.SaveAsync(m_context);
return(SealDirectory(pipConstructionHelper, downloadData, downloadData.ContentsFolder, sortedFiles));
}
catch (Exception e)
when(e is BuildXLException || e is IOException || e is UnauthorizedAccessException)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(EvaluationResult.Error);
}
}