/// <nodoc />
private static DirectoryArtifact SealDirectoryPartialOrFull(
PipConstructionHelper pipConstructionHelper,
AbsolutePath directoryRoot,
SealDirectoryKind kind,
FileArtifact[] contents,
string[] tags = null,
string description = null,
string[] patterns = null)
{
if (!pipConstructionHelper.TrySealDirectory(
directoryRoot,
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(
contents,
OrdinalFileArtifactComparer.Instance),
CollectionUtilities.EmptySortedReadOnlyArray <DirectoryArtifact, OrdinalDirectoryArtifactComparer>(OrdinalDirectoryArtifactComparer.Instance),
kind,
tags,
description,
patterns,
out var sealedDirectory
))
{
throw new InvalidOperationException();
}
return(sealedDirectory);
}
public void OutputExistenceAssertionsUnderOpaqueConsumptionBehavior(SealDirectoryKind kind)
{
var outputDirectory = CreateUniqueObjPath("outputDir");
var outputFile = CreateOutputFileArtifact(outputDirectory, "fileA");
var builderA = CreatePipBuilder(
new[]
{
Operation.WriteFile(outputFile, doNotInfer: true),
});
builderA.AddOutputDirectory(outputDirectory, kind);
var processA = SchedulePipBuilder(builderA);
// Assert the existence of the file under the opaque and consume it downstream
var result = PipConstructionHelper.TryAssertOutputExistenceInOpaqueDirectory(processA.ProcessOutputs.GetOpaqueDirectory(outputDirectory), outputFile, out var fileInOpaque);
XAssert.IsTrue(result);
var builderB = CreatePipBuilder(
new[]
{
Operation.ReadFile(fileInOpaque),
Operation.WriteFile(CreateOutputFileArtifact()),
});
SchedulePipBuilder(builderB);
RunScheduler().AssertSuccess();
}
public NinjaPipConstructor(FrontEndContext context, FrontEndHost frontEndHost, string frontEndName, ModuleDefinition moduleDefinition, QualifierId qualifierId, AbsolutePath projectRoot, AbsolutePath specPath, bool suppressDebugFlags, IUntrackingSettings untrackingSettings)
{
Contract.Requires(context != null);
Contract.Requires(frontEndHost != null);
Contract.Requires(moduleDefinition != null);
Contract.Requires(projectRoot.IsValid);
Contract.Requires(specPath.IsValid);
m_context = context;
m_frontEndHost = frontEndHost;
m_moduleDefinition = moduleDefinition;
m_projectRoot = projectRoot;
m_specPath = specPath;
m_suppressDebugFlags = suppressDebugFlags;
m_untrackingSettings = untrackingSettings;
m_pipConstructionHelper = GetPipConstructionHelperForModule(m_projectRoot, moduleDefinition, qualifierId);
m_frontEndName = frontEndName;
m_manuallyDroppedDependenciesPath = Lazy.Create(() => m_frontEndHost.Configuration.Layout.BuildEngineDirectory
.Combine(m_context.PathTable, RelativePath.Create(m_context.StringTable, @"tools\CMakeNinjaPipEnvironment")));
// Lazy initialization of environment variables and passthroughs
var allEnvironmentVariables = Lazy.Create(GetAllEnvironmentVariables);
m_environmentVariables = Lazy.Create(() => allEnvironmentVariables.Value.Where(kvp => SpecialEnvironmentVariables.PassThroughPrefixes.All(prefix => !kvp.Key.StartsWith(prefix))));
m_passThroughEnvironmentVariables = Lazy.Create(() => allEnvironmentVariables.Value.Where(kvp => SpecialEnvironmentVariables.PassThroughPrefixes.Any(prefix => kvp.Key.StartsWith(prefix))));
}
public void CompositeSharedOpaqueSubDirectoryReadBehavior(bool isValidAccess, bool createCompositeSharedOpaque)
{
/*
* sod1 <- composed sod (created if createCompositeSharedOpaque is true)
* sod2 <- original sod
* fileA
* subDir <- composed sub-directory
* fileB
*/
var root = Path.Combine(ObjectRoot, "sod1");
var sharedOpaqueDir = Path.Combine(root, "sod2");
var sharedOpaqueSubDir = Path.Combine(sharedOpaqueDir, "subDir");
var rootPath = AbsolutePath.Create(Context.PathTable, root);
var sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
var sharedOpaqueSubirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueSubDir);
var sharedOpaqueDirArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath);
var fileA = CreateOutputFileArtifact(sharedOpaqueDir);
var fileB = CreateOutputFileArtifact(sharedOpaqueSubDir);
var builderA = CreatePipBuilder(new Operation[]
{
Operation.WriteFile(fileA, doNotInfer: true),
Operation.WriteFile(fileB, doNotInfer: true)
});
builderA.AddOutputDirectory(sharedOpaqueDirArtifact, SealDirectoryKind.SharedOpaque);
var pipA = SchedulePipBuilder(builderA);
DirectoryArtifact subDirToConsume;
var producedSod = pipA.ProcessOutputs.GetOpaqueDirectory(sharedOpaqueDirPath);
if (createCompositeSharedOpaque)
{
// test that we can get a subDir from a composite sod
var success = PipConstructionHelper.TryComposeSharedOpaqueDirectory(rootPath, new[] { producedSod }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: null, description: null, tags: new string[] { }, out var composedSod);
XAssert.IsTrue(success);
success = PipConstructionHelper.TryComposeSharedOpaqueDirectory(sharedOpaqueSubirPath, new[] { composedSod }, SealDirectoryCompositionActionKind.NarrowDirectoryCone, contentFilter: null, description: null, tags: new string[] { }, out subDirToConsume);
XAssert.IsTrue(success);
}
else
{
var success = PipConstructionHelper.TryComposeSharedOpaqueDirectory(sharedOpaqueSubirPath, new[] { producedSod }, SealDirectoryCompositionActionKind.NarrowDirectoryCone, contentFilter: null, description: null, tags: new string[] { }, out subDirToConsume);
XAssert.IsTrue(success);
}
var builderB = CreateOpaqueDirectoryConsumer(CreateOutputFileArtifact(), null, subDirToConsume, isValidAccess ? fileB : fileA);
var pipB = SchedulePipBuilder(builderB);
if (isValidAccess)
{
RunScheduler().AssertSuccess();
}
else
{
IgnoreWarnings();
RunScheduler().AssertFailure();
AssertErrorEventLogged(LogEventId.FileMonitoringError);
}
}
public DirectoryArtifact SealDirectory(
AbsolutePath directoryRoot,
IReadOnlyList <FileArtifact> contents,
SealDirectoryKind kind,
string[] tags = null,
string description = null,
string[] patterns = null,
bool scrub = false)
{
if (!PipConstructionHelper.TrySealDirectory(
directoryRoot,
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(
contents,
OrdinalFileArtifactComparer.Instance),
CollectionUtilities.EmptySortedReadOnlyArray <DirectoryArtifact, OrdinalDirectoryArtifactComparer>(OrdinalDirectoryArtifactComparer.Instance),
kind,
tags,
description,
patterns,
out DirectoryArtifact sealedDirectory,
scrub))
{
throw new BuildXLTestException("Failed to add sealDirectory pip");
}
return(sealedDirectory);
}
public void DuplicateContentIsAllowedInCompositeSharedOpaqueDirectory()
{
CreatePip(
@"root\pipA",
out FileArtifact _,
out FileArtifact outputA,
out ProcessWithOutputs _,
out DirectoryArtifact soA);
var root = AbsolutePath.Create(Context.PathTable, Path.Combine(ObjectRoot, "root"));
// We construct a composite shared opaque that constains soA twice
var result = PipConstructionHelper.TryComposeSharedOpaqueDirectory(root, new[] { soA, soA }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: null, description: null, tags: new string[] { }, out var composedOpaque);
XAssert.IsTrue(result);
// PipB consumes the composed shared opaque and reads from outputA
var builder = CreatePipBuilder(new List <Operation>
{
Operation.ReadFile(outputA, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact())
});
builder.AddInputDirectory(composedOpaque);
var pipB = SchedulePipBuilder(builder);
IgnoreWarnings();
RunScheduler().AssertSuccess();
}
private bool TrySetBuildToolExecutor(
PipConstructionHelper pipConstructionHelper,
ProcessBuilder processBuilder,
ProjectWithPredictions project)
{
// If we should use the dotnet core version of msbuild, the executable for the pip is dotnet.exe instead of msbuild.exe, and
// the first argument is msbuild.dll
FileArtifact cmdExeArtifact;
if (m_resolverSettings.ShouldRunDotNetCoreMSBuild())
{
cmdExeArtifact = FileArtifact.CreateSourceFile(m_dotnetExePath);
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(m_msBuildPath));
}
else
{
cmdExeArtifact = FileArtifact.CreateSourceFile(m_msBuildPath);
}
processBuilder.Executable = cmdExeArtifact;
processBuilder.AddInputFile(cmdExeArtifact);
processBuilder.AddCurrentHostOSDirectories();
processBuilder.AddUntrackedAppDataDirectories();
processBuilder.AddUntrackedProgramDataDirectories();
// Temp directory setup including setting TMP and TEMP env vars. The path to
// the temp dir is generated in a consistent fashion between BuildXL runs to
// ensure environment value (and hence pip hash) consistency.
processBuilder.EnableTempDirectory();
processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {project.FullPath.ToString(PathTable)}"));
return(true);
}
/// <summary>
/// Schedules an IPC pip.
/// </summary>
public IpcPip ScheduleIpcPip(
IIpcMoniker moniker,
PipId?servicePipId,
ProcessBuilder ipcProcessBuilder,
FileArtifact outputFile,
bool isServiceFinalization,
PipConstructionHelper helper = null)
{
var ipcClientInfo = new IpcClientInfo(StringId.Create(Context.StringTable, moniker.Id), new ClientConfig(0, 0));
PipData arguments = ipcProcessBuilder.ArgumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
ReadOnlyArray <FileArtifact> fileDependencies = ipcProcessBuilder.GetInputFilesSoFar();
if (!(helper ?? m_defaultConstructionHelper).TryAddIpc(
ipcClientInfo,
arguments,
outputFile,
servicePipDependencies: servicePipId != null ? ReadOnlyArray <PipId> .From(new[] { servicePipId.Value }) : ReadOnlyArray <PipId> .Empty,
fileDependencies: fileDependencies,
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty,
skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .Empty,
isServiceFinalization: isServiceFinalization,
mustRunOnMaster: false,
tags: new string[0],
out IpcPip ipcPip))
{
throw new BuildXLTestException("Failed to add ipc pip");
}
return(ipcPip);
}
public NinjaPipConstructor(FrontEndContext context,
FrontEndHost frontEndHost,
string frontEndName,
ModuleDefinition moduleDefinition,
QualifierId qualifierId,
AbsolutePath projectRoot,
AbsolutePath specPath, bool suppressDebugFlags,
IEnumerable <KeyValuePair <string, string> > userDefinedEnvironment,
IEnumerable <string> userDefinedPassthroughVariables,
IUntrackingSettings untrackingSettings)
{
Contract.Requires(context != null);
Contract.Requires(frontEndHost != null);
Contract.Requires(moduleDefinition != null);
Contract.Requires(projectRoot.IsValid);
Contract.Requires(specPath.IsValid);
m_context = context;
m_frontEndHost = frontEndHost;
m_moduleDefinition = moduleDefinition;
m_projectRoot = projectRoot;
m_specPath = specPath;
m_suppressDebugFlags = suppressDebugFlags;
m_untrackingSettings = untrackingSettings;
m_pipConstructionHelper = GetPipConstructionHelperForModule(m_projectRoot, moduleDefinition, qualifierId);
m_frontEndName = frontEndName;
m_manuallyDroppedDependenciesPath = m_frontEndHost.Configuration.Layout.BuildEngineDirectory
.Combine(m_context.PathTable, RelativePath.Create(m_context.StringTable, @"tools\CMakeNinjaPipEnvironment"));
PrepareEnvironment(userDefinedEnvironment, userDefinedPassthroughVariables, out m_userDefinedEnvironment, out m_userDefinedPassthroughVariables);
}
public void CompositeContentIsProperlyMaterialized()
{
// This test verifies that BXL properly materializes composite shared opaques
// (i.e., the files are materialized because we need to materialize the composite opaque
// and not because we somehow materialized the original shared opaque directory).
var root = Path.Combine(ObjectRoot, "root");
var rootPath = AbsolutePath.Create(Context.PathTable, root);
var sharedOpaqueDir = Path.Combine(ObjectRoot, "root", "CreateOutputFileArtifact");
var sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
var inputB = CreateSourceFile();
var outputA = CreateOutputFileArtifact(sharedOpaqueDir);
// PipA - produces a dynamic file under /root/CreateOutputFileArtifact
var builderA = CreatePipBuilder(new Operation[]
{
Operation.WriteFile(outputA, doNotInfer: true)
});
builderA.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath), SealDirectoryKind.SharedOpaque);
builderA.AddTags(Context.StringTable, "pipA");
var pipA = SchedulePipBuilder(builderA);
// composite shared opaque - consists of a single shared opaque produced by PipA
var success = PipConstructionHelper.TryComposeSharedOpaqueDirectory(rootPath, new[] { pipA.ProcessOutputs.GetOpaqueDirectory(sharedOpaqueDirPath) }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: null, description: null, tags: new string[] { }, out var composedOpaque);
XAssert.IsTrue(success);
// PipB - consumes composite shared opaque directory
// note: there is no direct dependency on PipA
var builderB = CreatePipBuilder(new Operation[]
{
Operation.ReadRequiredFile(outputA, doNotInfer: true), // dynamic output of PipA
Operation.ReadFile(inputB), // dummy input, so we can force pipB to re-run
Operation.WriteFile(CreateOutputFileArtifact())
});
builderB.AddInputDirectory(composedOpaque);
builderB.AddTags(Context.StringTable, "pipB");
var pipB = SchedulePipBuilder(builderB);
// bring content into the cache
RunScheduler().AssertCacheMiss(pipA.Process.PipId, pipB.Process.PipId);
// Lazy materialization and tags are used here to prevent materialization of opaque directory produced by pipA,
// so the only place where 'outputA' might come from is materialization of 'composedOpaque'.
Configuration.Schedule.EnableLazyOutputMaterialization = true;
Configuration.Filter = "tag='pipB'";
// make sure that we start with no files materialized
Directory.Delete(root, recursive: true);
// force pipB to re-run
File.AppendAllText(ArtifactToString(inputB), "foo");
RunScheduler().AssertSuccess().AssertCacheHit(pipA.Process.PipId).AssertCacheMiss(pipB.Process.PipId);
}
/// <summary>
/// Pip consumes a filtered SOD (fileA, fileB) and reads both files
/// Filter is changed -> SOD (fileA)
/// Pip should be a miss and should emit DFA
/// </summary>
public void FilterChangesShouldAffectCachingIfPipReadsFilteredFile()
{
var includeKind = global::BuildXL.Pips.Operations.SealDirectoryContentFilter.ContentFilterKind.Include;
var sharedOpaqueDir = Path.Combine(ObjectRoot, "sod");
var sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
var sharedOpaqueDirArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(sharedOpaqueDirPath);
var fileA = CreateOutputFileArtifact(sharedOpaqueDir);
var fileB = CreateOutputFileArtifact(sharedOpaqueDir);
var builderA = CreatePipBuilder(new Operation[]
{
Operation.WriteFile(fileA, doNotInfer: true),
Operation.WriteFile(fileB, doNotInfer: true)
});
builderA.AddOutputDirectory(sharedOpaqueDirArtifact, SealDirectoryKind.SharedOpaque);
var pipA = SchedulePipBuilder(builderA);
var producedSod = pipA.ProcessOutputs.GetOpaqueDirectory(sharedOpaqueDirPath);
var success = PipConstructionHelper.TryComposeSharedOpaqueDirectory(sharedOpaqueDirPath, new[] { producedSod }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: new SealDirectoryContentFilter(includeKind, ".*"), description: null, tags: new string[] { }, out var filteredOpaque);
XAssert.IsTrue(success);
var builderB = CreateOpaqueDirectoryConsumer(CreateOutputFileArtifact(), null, filteredOpaque, fileA, fileB);
var pipB = SchedulePipBuilder(builderB);
// first run - populate cache
RunScheduler().AssertSuccess();
// reset the graph and this time specify a filter that would exclude fileB
ResetPipGraphBuilder();
pipA = SchedulePipBuilder(builderA);
producedSod = pipA.ProcessOutputs.GetOpaqueDirectory(sharedOpaqueDirPath);
var filter = $".*{fileA.Path.GetName(Context.PathTable).ToString(Context.StringTable)}$";
success = PipConstructionHelper.TryComposeSharedOpaqueDirectory(sharedOpaqueDirPath, new[] { producedSod }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: new SealDirectoryContentFilter(includeKind, filter), description: null, tags: new string[] { }, out filteredOpaque);
XAssert.IsTrue(success);
builderB = CreateOpaqueDirectoryConsumer(CreateOutputFileArtifact(), null, filteredOpaque, fileA, fileB);
pipB = SchedulePipBuilder(builderB);
// since filteredOpaque does not contain fileB, this run should result in a DFA
IgnoreWarnings();
var result = RunScheduler().AssertFailure();
AssertErrorEventLogged(LogEventId.FileMonitoringError);
result
// pipA should be a hit
.AssertCacheHitWithoutAssertingSuccess(pipA.Process.PipId)
// pipB should be a miss (access to fileB is no longer valid)
.AssertCacheMissWithoutAssertingSuccess(pipB.Process.PipId)
// pipB should have a weak FP hit (file accesses are checked during strong FP computation)
.AssertNumWeakFingerprintMisses(0);
}
/// <nodoc />
public static ProcessOutputs AddProcess(
this PipConstructionHelper pipConstructionHelper,
ProcessBuilder builder)
{
if (!pipConstructionHelper.TryAddProcess(builder, out var outputs, out _))
{
throw new InvalidOperationException();
}
return(outputs);
}
/// <summary>
/// Creates an instance of <see cref="TestPipGraphFragment"/>.
/// </summary>
public TestPipGraphFragment(LoggingContext loggingContext, string sourceRoot, string objectRoot, string redirectedRoot, string moduleName, bool useTopSort = false)
{
Contract.Requires(loggingContext != null);
Contract.Requires(!string.IsNullOrEmpty(sourceRoot));
Contract.Requires(!string.IsNullOrEmpty(objectRoot));
Contract.Requires(!string.IsNullOrEmpty(moduleName));
Context = BuildXLContext.CreateInstanceForTesting();
m_loggingContext = loggingContext;
m_sourceRoot = AbsolutePath.Create(Context.PathTable, sourceRoot);
m_objectRoot = AbsolutePath.Create(Context.PathTable, objectRoot);
m_expander = new MountPathExpander(Context.PathTable);
var configuration = new ConfigurationImpl()
{
Schedule =
{
UseFixedApiServerMoniker = true,
ComputePipStaticFingerprints = true,
}
};
m_useTopSort = useTopSort;
PipGraph = m_useTopSort
? new PipGraphFragmentBuilderTopSort(Context, configuration, m_expander)
: new PipGraphFragmentBuilder(Context, configuration, m_expander);
ModuleName = moduleName;
var specFileName = moduleName + ".dsc";
m_specPath = m_sourceRoot.Combine(Context.PathTable, specFileName);
m_moduleId = ModuleId.Create(StringId.Create(Context.StringTable, moduleName));
var modulePip = ModulePip.CreateForTesting(
Context.StringTable,
m_specPath,
m_moduleId);
PipGraph.AddModule(modulePip);
PipGraph.AddSpecFile(new SpecFilePip(new FileArtifact(m_specPath), new LocationData(m_specPath, 0, 0), modulePip.Module));
m_defaultConstructionHelper = PipConstructionHelper.CreateForTesting(
Context,
objectRoot: m_objectRoot,
redirectedRoot: AbsolutePath.Create(Context.PathTable, redirectedRoot),
pipGraph: PipGraph,
moduleName: moduleName,
specRelativePath: Path.Combine(m_sourceRoot.GetName(Context.PathTable).ToString(Context.StringTable), specFileName),
specPath: m_specPath,
symbol: moduleName + "_defaultValue");
}
/// <nodoc />
private async Task <EvaluationResult> CheckIfExtractIsNeededAsync(PipConstructionHelper pipConstructionHelper, DownloadData downloadData)
{
try
{
if (m_context.FileSystem.IsDirectory(downloadData.ContentsFolder))
{
var incrementalState = await ExtractIncrementalState.TryLoadAsync(m_logger, m_context, downloadData);
if (incrementalState != null)
{
// Check all files still have the same hash. This should use the hash cache based on USN so be very fast.
foreach (var hashKv in incrementalState.Hashes)
{
if (!m_context.FileSystem.Exists(hashKv.Key))
{
// File is not present, extraction is needed.
return(EvaluationResult.Continue);
}
var hash = await GetContentHashAsync(hashKv.Key);
if (hash != hashKv.Value)
{
// File has changed, extraction is needed.
return(EvaluationResult.Continue);
}
}
// All hashes verified, update the manifest
await incrementalState.SaveAsync(m_context);
return(SealDirectory(pipConstructionHelper, downloadData, downloadData.ContentsFolder, incrementalState.Files));
}
}
}
catch (IOException e)
{
m_logger.ErrorCheckingIncrementality(m_context.LoggingContext, downloadData.Settings.ModuleName, e.Message);
return(EvaluationResult.Error);
}
catch (UnauthorizedAccessException e)
{
m_logger.ErrorCheckingIncrementality(m_context.LoggingContext, downloadData.Settings.ModuleName, e.Message);
return(EvaluationResult.Error);
}
// Extraction is needed
return(EvaluationResult.Continue);
}
public void CompositeSharedOpaqueDirectoryWithFilterReadBehavior(bool validFileAccess)
{
var includeKind = global::BuildXL.Pips.Operations.SealDirectoryContentFilter.ContentFilterKind.Include;
CreatePip(
@"root\pipA",
out FileArtifact _,
out FileArtifact outputA,
out ProcessWithOutputs pipA,
out DirectoryArtifact soA);
CreatePip(
@"root\pipB",
out FileArtifact _,
out FileArtifact outputB,
out ProcessWithOutputs _,
out DirectoryArtifact soB);
var root = AbsolutePath.Create(Context.PathTable, Path.Combine(ObjectRoot, "root"));
// We construct a composite shared opaque using both soA and soB, but apply the filter that matches only soA
var filter = $".*{outputA.Path.GetName(Context.PathTable).ToString(Context.StringTable)}$";
var result = PipConstructionHelper.TryComposeSharedOpaqueDirectory(root, new[] { soA, soB }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: new SealDirectoryContentFilter(includeKind, filter), description: null, tags: new string[] { }, out var composedOpaque);
XAssert.IsTrue(result);
// PipC consumes the composed shared opaque and reads a file
// validFileAccess == true => outputA => allowed access
// validFileAccess == false => outputB => DFA (outputB does not match the filter, hence it's not a part of a composite SOD)
var builder = CreatePipBuilder(new List <Operation>
{
Operation.ReadFile(validFileAccess ? outputA : outputB, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact())
});
builder.AddInputDirectory(composedOpaque);
var pipC = SchedulePipBuilder(builder);
if (validFileAccess)
{
RunScheduler().AssertSuccess();
}
else
{
IgnoreWarnings();
RunScheduler().AssertFailure();
AssertErrorEventLogged(LogEventId.FileMonitoringError);
}
}
public void OutputExistenceAssertionsUnderOpaqueCachingBehavior(SealDirectoryKind kind)
{
var outputDirectory = CreateUniqueObjPath("outputDir");
var outputFile = CreateOutputFileArtifact(outputDirectory, "fileA");
var builderA = CreatePipBuilder(
new[]
{
Operation.WriteFile(outputFile, doNotInfer: true),
});
builderA.AddOutputDirectory(outputDirectory, kind);
var processA = SchedulePipBuilder(builderA);
// Assert the existence of the file under the opaque and consume it downstream
var result = PipConstructionHelper.TryAssertOutputExistenceInOpaqueDirectory(processA.ProcessOutputs.GetOpaqueDirectory(outputDirectory), outputFile, out var fileInOpaque);
XAssert.IsTrue(result);
var builderB = CreatePipBuilder(
new[]
{
Operation.ReadFile(fileInOpaque),
Operation.WriteFile(CreateOutputFileArtifact()),
});
var processB = SchedulePipBuilder(builderB);
RunScheduler(runNameOrDescription: "First build").AssertSuccess();
// Second run should be a cache hit
RunScheduler(runNameOrDescription: "Second build - all cached").AssertCacheHit(processA.Process.PipId, processB.Process.PipId);
AssertVerboseEventLogged(LogEventId.ProcessPipCacheHit, 2);
ResetPipGraphBuilder();
processA = SchedulePipBuilder(builderA);
// Assert a non-existent file
result = PipConstructionHelper.TryAssertOutputExistenceInOpaqueDirectory(processA.ProcessOutputs.GetOpaqueDirectory(outputDirectory), CreateOutputFileArtifact(outputDirectory), out _);
XAssert.IsTrue(result);
// We should detect the non existent file
RunScheduler(runNameOrDescription: "Third build - cached but existence assertion failed").AssertFailure();
AssertErrorEventLogged(global::BuildXL.Processes.Tracing.LogEventId.ExistenceAssertionUnderOutputDirectoryFailed);
// But the process pip should have been a cache hit anyway
AssertVerboseEventLogged(LogEventId.ProcessPipCacheHit);
}
/// <nodoc />
public static DirectoryArtifact SealDirectoryPartial(
this PipConstructionHelper pipConstructionHelper,
AbsolutePath directoryRoot,
FileArtifact[] contents,
string[] tags = null,
string description = null)
{
return(SealDirectoryPartialOrFull(
pipConstructionHelper,
directoryRoot,
SealDirectoryKind.Partial,
contents,
tags,
description,
patterns: null));
}
public void TrackerLoadFailureShouldNotResultInUnderbuild()
{
AbsolutePath path = CreateUniqueSourcePath().Combine(Context.PathTable, "MySrc");
FileArtifact srcA = CreateSourceFile(path);
// Populate file content table with srcA's content via P.
var pOperations = new[] { Operation.ReadFile(srcA), Operation.WriteFile(CreateOutputFileArtifact()) };
var p = CreateAndSchedulePipBuilder(pOperations);
var result = RunScheduler().AssertCacheMiss(p.Process.PipId);
// Modify srcA.
File.AppendAllText(ArtifactToString(srcA), Guid.NewGuid().ToString());
// Destroy file change tracker.
FileUtilities.DeleteFile(result.Config.Layout.SchedulerFileChangeTrackerFile.ToString(Context.PathTable), waitUntilDeletionFinished: true);
ResetPipGraphBuilder();
// Make srcA tracked and probe via Q.
var ssd = PipConstructionHelper.SealDirectorySource(path);
var qOperations = new[] { Operation.Probe(srcA, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) };
var qBuilder = CreatePipBuilder(qOperations);
qBuilder.AddInputDirectory(ssd);
var q = SchedulePipBuilder(qBuilder);
RunScheduler().AssertCacheMiss(q.Process.PipId);
// Modify basic file info.
var creationTime = File.GetCreationTime(ArtifactToString(srcA));
File.SetCreationTime(ArtifactToString(srcA), creationTime.Add(TimeSpan.FromSeconds(10)));
// Switch back to P.
ResetPipGraphBuilder();
p = CreateAndSchedulePipBuilder(pOperations);
// P should be miss because the content of srcA changed.
RunScheduler().AssertCacheMiss(p.Process.PipId);
}
public FileArtifact WriteFile(AbsolutePath destination, PipDataAtom content, WriteFileEncoding?encoding = null)
{
PipDataBuilder pipDataBuilder = new PipDataBuilder(Context.StringTable);
pipDataBuilder.Add(content);
if (!PipConstructionHelper.TryWriteFile(
destination,
pipDataBuilder.ToPipData(Context.StringTable.Empty, PipDataFragmentEscaping.NoEscaping),
encoding ?? WriteFileEncoding.Utf8,
null,
null,
out var result))
{
throw new BuildXLTestException("Failed to add writefile pip");
}
return(result);
}
private EvaluationResult SealDirectory(PipConstructionHelper pipConstructionHelper, DownloadData downloadData, DirectoryArtifact directory, SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> files)
{
if (!pipConstructionHelper.TrySealDirectory(
directory,
files,
Pips.Operations.SealDirectoryKind.Partial,
null,
null,
null,
out var directoryArtifact)
)
{
return(EvaluationResult.Error);
}
var staticDirectory = new StaticDirectory(directoryArtifact, Pips.Operations.SealDirectoryKind.Partial, files.WithCompatibleComparer(OrdinalPathOnlyFileArtifactComparer.Instance));
return(new EvaluationResult(staticDirectory));
}
public void CompositeSharedOpaqueDirectoryConsumptionCachingBehavior()
{
CreatePip(
@"root\pipA",
out FileArtifact inputA,
out FileArtifact outputA,
out ProcessWithOutputs pipA,
out DirectoryArtifact soA);
CreatePip(
@"root\pipB",
out FileArtifact inputB,
out FileArtifact outputB,
out ProcessWithOutputs pipB,
out DirectoryArtifact soB);
var root = AbsolutePath.Create(Context.PathTable, Path.Combine(ObjectRoot, "root"));
// We create a composite shared opaque containing both shared opaques
var result = PipConstructionHelper.TryComposeSharedOpaqueDirectory(root, new[] { soA, soB }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: null, description: null, tags: new string[] { }, out var composedOpaque);
XAssert.IsTrue(result);
// PipC consumes the composed shared opaque and reads from both outputs
var builder = CreatePipBuilder(new List <Operation>
{
Operation.ReadFile(outputA, doNotInfer: true),
Operation.ReadFile(outputB, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact())
});
builder.AddInputDirectory(composedOpaque);
var pipC = SchedulePipBuilder(builder);
// First time all should miss, second time all should hit
RunScheduler().AssertCacheMiss(pipA.Process.PipId, pipB.Process.PipId, pipC.Process.PipId).AssertSuccess();
RunScheduler().AssertCacheHit(pipA.Process.PipId, pipB.Process.PipId, pipC.Process.PipId);
// Modify inputA and make sure that only pipA and PipC re-run
File.WriteAllText(ArtifactToString(inputA), "New content");
RunScheduler().AssertCacheMiss(pipA.Process.PipId, pipC.Process.PipId).AssertCacheHit(pipB.Process.PipId);
RunScheduler().AssertCacheHit(pipA.Process.PipId, pipB.Process.PipId, pipC.Process.PipId);
}
public void OutputExistenceAssertionsUnderOpaqueIsValidated(SealDirectoryKind kind)
{
var outputDirectory = CreateUniqueObjPath("outputDir");
var outputFile = CreateOutputFileArtifact(outputDirectory, "fileA");
// A do-nothing pip
var builderA = CreatePipBuilder(new Operation[] {});
builderA.AddOutputDirectory(outputDirectory, kind);
var processA = SchedulePipBuilder(builderA);
// Assert the existence of the file under the opaque
var result = PipConstructionHelper.TryAssertOutputExistenceInOpaqueDirectory(processA.ProcessOutputs.GetOpaqueDirectory(outputDirectory), outputFile, out var fileInOpaque);
XAssert.IsTrue(result);
// The file is not produced, we should detect it and fail
RunScheduler().AssertFailure();
AssertErrorEventLogged(global::BuildXL.Processes.Tracing.LogEventId.ExistenceAssertionUnderOutputDirectoryFailed);
}
public FileArtifact CopyFile(
FileArtifact source,
AbsolutePath destination,
string description = null,
string[] tags = null,
CopyFile.Options options = default)
{
if (!PipConstructionHelper.TryCopyFile(
source,
destination,
global::BuildXL.Pips.Operations.CopyFile.Options.None,
tags,
description,
out var result))
{
throw new BuildXLTestException("Failed to add copyfile pip");
}
return(result);
}
public void CompositeSharedOpaqueDirectoryReadBehavior()
{
CreatePip(
@"root\pipA",
out FileArtifact _,
out FileArtifact outputA,
out ProcessWithOutputs _,
out DirectoryArtifact soA);
CreatePip(
@"root\pipB",
out FileArtifact _,
out FileArtifact outputB,
out ProcessWithOutputs _,
out DirectoryArtifact soB);
var root = AbsolutePath.Create(Context.PathTable, Path.Combine(ObjectRoot, "root"));
// We construct a composite shared opaque that only contains soa (and *not* sob)
var result = PipConstructionHelper.TryComposeSharedOpaqueDirectory(root, new[] { soA }, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter: null, description: null, tags: new string[] { }, out var composedOpaque);
XAssert.IsTrue(result);
// PipC consumes the composed shared opaque and reads from both outputs
var builder = CreatePipBuilder(new List <Operation>
{
Operation.ReadFile(outputA, doNotInfer: true),
Operation.ReadFile(outputB, doNotInfer: true), // this one should be disallowed
Operation.WriteFile(CreateOutputFileArtifact())
});
builder.AddInputDirectory(composedOpaque);
var pipC = SchedulePipBuilder(builder);
IgnoreWarnings();
// pipC should not be allowed to read from outputB, since the composed shared opaque does not contain it
RunScheduler().AssertFailure();
AssertErrorEventLogged(LogEventId.FileMonitoringError);
}
/// <nodoc />
public static DirectoryArtifact SealDirectorySource(
this PipConstructionHelper pipConstructionHelper,
AbsolutePath directoryRoot,
SealDirectoryKind kind = SealDirectoryKind.SourceAllDirectories,
string[] tags = null,
string description = null,
string[] patterns = null)
{
if (!pipConstructionHelper.TrySealDirectory(
directoryRoot,
CollectionUtilities.EmptySortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer>(OrdinalFileArtifactComparer.Instance),
kind,
tags,
description,
patterns,
out var sealedDirectory
))
{
throw new InvalidOperationException();
}
return(sealedDirectory);
}
private PipConstructionHelper GetPipConstructionHelperForModule(AbsolutePath projectRoot, ModuleDefinition moduleDefinition, QualifierId qualifierId)
{
// One and only one AbsolutePath in the specs (corresponding to the build.ninja
// If this changed, this method would be out of here, as we would need a different PipConstructionHelper
// for each spec
Contract.Requires(moduleDefinition.Specs.Count == 1);
// Get a symbol that is unique for this particular instance
var fullSymbol = FullSymbol.Create(m_context.SymbolTable, $"ninja.{PipConstructionUtilities.SanitizeStringForSymbol(moduleDefinition.Descriptor.Name)}"); // TODO: Figure this out, complete
AbsolutePath pathToSpec = moduleDefinition.Specs.First();
if (!projectRoot.TryGetRelative(m_context.PathTable, pathToSpec, out var specRelativePath))
{
// Issue a warning and continue with Invalid path. PipConstructionHelper will just ignore
Tracing.Logger.Log.CouldNotComputeRelativePathToSpec(m_context.LoggingContext,
Location.FromFile(projectRoot.ToString(m_context.PathTable)),
projectRoot.ToString(m_context.PathTable),
pathToSpec.ToString(m_context.PathTable));
specRelativePath = RelativePath.Invalid;
}
var pipConstructionHelper = PipConstructionHelper.Create(
m_context,
m_frontEndHost.Engine.Layout.ObjectDirectory,
m_frontEndHost.Engine.Layout.RedirectedDirectory,
m_frontEndHost.Engine.Layout.TempDirectory,
m_frontEndHost.PipGraph,
moduleDefinition.Descriptor.Id,
moduleDefinition.Descriptor.Name,
specRelativePath,
fullSymbol,
new LocationData(pathToSpec, 0, 0), // TODO: This is the location of the value (that is scheduling pips through this helper) in its corresponding spec.
// Since we are not exposing any public value yet that represents this symbol, this location is fine.
qualifierId);
return(pipConstructionHelper);
}
private PipConstructionHelper GetPipConstructionHelperForProject(ProjectWithPredictions project, QualifierId qualifierId)
{
var pathToProject = project.FullPath;
// We might be adding the same spec file pip more than once when the same project is evaluated
// under different global properties, but that's fine, the pip graph ignores duplicates
m_frontEndHost.PipGraph.AddSpecFile(
new SpecFilePip(
FileArtifact.CreateSourceFile(pathToProject),
new LocationData(pathToProject, 0, 0),
m_moduleDefinition.Descriptor.Id));
Root.TryGetRelative(PathTable, pathToProject, out var specRelativePath);
if (!PathAtom.TryCreate(m_context.StringTable, m_moduleDefinition.Descriptor.Name, out _))
{
throw new ArgumentException($"Failed to create PathAtom from {m_moduleDefinition.Descriptor.Name}");
}
// Get a symbol that is unique for this particular project instance
var fullSymbol = GetFullSymbolFromProject(project);
var pipConstructionHelper = PipConstructionHelper.Create(
m_context,
m_frontEndHost.Engine.Layout.ObjectDirectory,
m_frontEndHost.Engine.Layout.RedirectedDirectory,
m_frontEndHost.Engine.Layout.TempDirectory,
m_frontEndHost.PipGraph,
m_moduleDefinition.Descriptor.Id,
m_moduleDefinition.Descriptor.Name,
specRelativePath,
fullSymbol,
new LocationData(pathToProject, 0, 0),
qualifierId);
return(pipConstructionHelper);
}
private bool TrySetBuildToolExecutor(
PipConstructionHelper pipConstructionHelper,
ProcessBuilder processBuilder,
ProjectWithPredictions project)
{
FileArtifact cmdExeArtifact = FileArtifact.CreateSourceFile(m_msBuildExePath);
processBuilder.Executable = cmdExeArtifact;
processBuilder.AddInputFile(cmdExeArtifact);
processBuilder.AddCurrentHostOSDirectories();
processBuilder.AddUntrackedAppDataDirectories();
processBuilder.AddUntrackedProgramDataDirectories();
// Temp directory setup including setting TMP and TEMP env vars. The path to
// the temp dir is generated in a consistent fashion between BuildXL runs to
// ensure environment value (and hence pip hash) consistency.
processBuilder.EnableTempDirectory();
AbsolutePath toolDir = m_msBuildExePath.GetParent(PathTable);
processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {project.FullPath.ToString(PathTable)}"));
return(true);
}
/// <summary>
/// Gets an IPC moniker.
/// </summary>
public IIpcMoniker GetIpcMoniker(PipConstructionHelper helper = null)
{
var semiStableHash = (helper ?? m_defaultConstructionHelper).GetNextSemiStableHash();
return(IpcFactory.GetProvider().LoadOrCreateMoniker(string.Format(CultureInfo.InvariantCulture, "{0:X16}", semiStableHash)));
}
/// <summary>
/// Schedules a process builder.
/// </summary>
public (Process process, ProcessOutputs outputs) ScheduleProcessBuilder(ProcessBuilder builder, PipConstructionHelper pipConstructionHelper = null)
{
var helper = pipConstructionHelper ?? m_defaultConstructionHelper;
if (!helper.TryAddProcess(builder, out ProcessOutputs outputs, out Process process))
{
throw new BuildXLTestException("Failed to add process pip");
}
return(process, outputs);
}
