public void ValidateCachingAbsentDirectoryProbes(bool sourceMount)
{
// Set up absent directory
DirectoryArtifact absentDirectory;
if (sourceMount)
{
// Source mounts (i.e. read only mounts) use the actual filesystem and check the existence of files/directories on disk
absentDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(ReadonlyRoot));
Directory.Delete(ArtifactToString(absentDirectory)); // start with absent directory
}
else
{
// Output mounts (i.e. read/write mounts) use the graph filesystem and do not check the existence of files/directories on disk
absentDirectory = CreateOutputDirectoryArtifact();
}
// Pip probes absent input and directory
Process pip = CreateAndSchedulePipBuilder(new Operation[]
{
Operation.Probe(absentDirectory),
Operation.WriteFile(CreateOutputFileArtifact())
}).Process;
RunScheduler().AssertScheduled(pip.PipId).AssertCacheMiss(pip.PipId);
RunScheduler().AssertNotScheduled(pip.PipId);
// Create /absentDirectory
Directory.CreateDirectory(ArtifactToString(absentDirectory));
// Source mounts check the existence of files/directories on disk (so cache miss)
// Output mounts do not check the existence of files/directories on disk (so cache hit)
if (sourceMount)
{
RunScheduler().AssertScheduled(pip.PipId).AssertCacheMiss(pip.PipId);
}
RunScheduler().AssertNotScheduled(pip.PipId);
// Create /absentDirectory/newFile
CreateSourceFile(ArtifactToString(absentDirectory));
RunScheduler().AssertNotScheduled(pip.PipId);
}
public void IncrementalSchedulingIsRobustAgainstWildDirectoryMembershipChangeBeforePipExecution(bool changeMembershipBeforeThirdRun)
{
var directoryPath = CreateUniqueDirectory(ReadonlyRoot);
FileArtifact file = CreateSourceFile(directoryPath);
FileArtifact inputA = CreateSourceFile();
FileArtifact outputA = CreateOutputFileArtifact();
DirectoryArtifact enumeratedDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(directoryPath);
var builderA = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(inputA),
Operation.EnumerateDir(enumeratedDirectory, doNotInfer: true),
Operation.WriteFile(outputA)
});
var pipA = SchedulePipBuilder(builderA);
RunScheduler().AssertScheduled(pipA.Process.PipId);
ModifyFile(inputA);
// Due to inputA modification, pipA becomes dirty.
RunScheduler(
testHooks: new SchedulerTestHooks()
{
IncrementalSchedulingStateAfterJournalScanAction = _ =>
{
// Change directory membership before pip execution.
CreateSourceFile(directoryPath);
}
}).AssertScheduled(pipA.Process.PipId);
AssertVerboseEventLogged(StorageLogEventId.ConflictDirectoryMembershipFingerprint, count: 1);
if (changeMembershipBeforeThirdRun)
{
// Change directory membership before third run.
CreateSourceFile(directoryPath);
}
// Due to membership change, pipA becomes dirty.
RunScheduler().AssertScheduled(pipA.Process.PipId);
}
public void ValidateCreatingDirectoryRetracksDirectoriesNeededForTrackedChildAbsentPaths()
{
// Set up absent file
AbsolutePath readOnlyRoot = AbsolutePath.Create(Context.PathTable, ReadonlyRoot);
DirectoryArtifact dir = SealDirectory(readOnlyRoot, SealDirectoryKind.SourceAllDirectories);
// Pip probes absent input and directory
var ops = new Operation[]
{
ProbeOp(ReadonlyRoot),
ProbeOp(ReadonlyRoot, @"dir1\a\dir1_a.txt"),
ProbeOp(ReadonlyRoot, @"dir1\b\dir1_b.txt"),
Operation.WriteFile(CreateOutputFileArtifact())
};
var builder = CreatePipBuilder(ops);
builder.AddInputDirectory(dir);
Process pip = SchedulePipBuilder(builder).Process;
RunScheduler().AssertCacheMiss(pip.PipId);
// Create probed file path which with a parent directory
// which is also a parent of another absent path probe
CreateDir(ReadonlyRoot, @"dir1");
CreateDir(ReadonlyRoot, @"dir1\a");
CreateFile(ReadonlyRoot, @"dir1\a\dir1_a.txt");
RunScheduler().AssertCacheMiss(pip.PipId);
// Now validate that creating a file at the location of
// the other absent path probe invalidates the pip
// In the original bug, the parent directory becomes untracked such that
// changes to the directory go unnoticed
CreateDir(ReadonlyRoot, @"dir1");
CreateDir(ReadonlyRoot, @"dir1\b");
CreateFile(ReadonlyRoot, @"dir1\b\dir1_b.txt");
RunScheduler().AssertCacheMiss(pip.PipId);
}
/// <inheritdoc />
protected override EvaluationResult DoEval(Context context, ModuleLiteral env, EvaluationStackFrame frame)
{
var evaluatedPathExpression = PathExpression.Eval(context, env, frame);
if (evaluatedPathExpression.IsErrorValue)
{
return(EvaluationResult.Error);
}
try
{
var path = Converter.ExpectPath(evaluatedPathExpression, strict: true, context: new ConversionContext(pos: 1));
return(EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(path)));
}
catch (ConvertException e)
{
context.Errors.ReportUnexpectedValueTypeOnConversion(env, e, Location);
return(EvaluationResult.Error);
}
}
private static DirectoryArtifact GetSpecialFolder(PathTable pathTable, Environment.SpecialFolder specialFolder, params string[] subFolders)
{
// GetFolderPath will return empty paths for special folders that don't exist in the current user profile.
// Return DirectoryArtifact.Invalid for those folders so they can be omitted from being untracked when
// the system does not support them.
if (AbsolutePath.TryCreate(pathTable, SpecialFolderUtilities.GetFolderPath(specialFolder), out var root))
{
if (subFolders != null)
{
foreach (var subFolder in subFolders)
{
root = root.Combine(pathTable, subFolder);
}
}
return(DirectoryArtifact.CreateWithZeroPartialSealId(root));
}
return(DirectoryArtifact.Invalid);
}
/// <summary>
/// Associates a directory artifact with its constituent file artifacts. The contents must be set before the directory artifact may
/// be used by a pip (see <see cref="ListSealedDirectoryContents"/>).
/// </summary>
public void SetSealedDirectoryContents(DirectoryArtifact directory, params FileArtifact[] artifactsInDirectory)
{
Contract.Requires(directory.IsValid);
Contract.Requires(artifactsInDirectory != null);
FileArtifact[] artifacts = artifactsInDirectory.ToArray();
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> sortedArtifacts = SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .SortUnsafe(
artifacts,
OrdinalFileArtifactComparer.Instance);
bool added = m_knownSealedDirectoryContents.TryAdd(directory, sortedArtifacts);
Contract.Assume(added);
foreach (FileArtifact artifact in artifacts)
{
Contract.Assume(artifact.Path.IsWithin(Context.PathTable, directory.Path));
m_knownSealedArtifacts[artifact.Path] = artifact;
}
}
public void DirectoryEnumerationUnderWritableMount()
{
var sealDirectoryPath = CreateUniqueDirectory(ObjectRoot);
var path = sealDirectoryPath.ToString(Context.PathTable);
DirectoryArtifact dir = SealDirectory(sealDirectoryPath, SealDirectoryKind.Partial, CreateSourceFile(path));
var ops = new Operation[]
{
Operation.EnumerateDir(dir),
Operation.WriteFile(CreateOutputFileArtifact())
};
var builder = CreatePipBuilder(ops);
Process pip = SchedulePipBuilder(builder).Process;
var result = RunScheduler().AssertSuccess();
result.AssertObservation(pip.PipId, new ObservedPathEntry(sealDirectoryPath, false, true, true, RegexDirectoryMembershipFilter.AllowAllRegex, false));
}
/// <nodoc />
public bool TrySealDirectory(
AbsolutePath directoryRoot,
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> contents,
SealDirectoryKind kind,
string[] tags,
string description,
string[] patterns,
out DirectoryArtifact sealedDirectory,
bool scrub = false)
{
Contract.Requires(directoryRoot.IsValid);
Contract.Requires(contents.IsValid);
PipData usage = PipDataBuilder.CreatePipData(Context.StringTable, string.Empty, PipDataFragmentEscaping.NoEscaping, description != null
? new PipDataAtom[] { description }
: new PipDataAtom[] { "'", directoryRoot, "' [", contents.Length.ToString(CultureInfo.InvariantCulture), " files]" });
var pip = new SealDirectory(
directoryRoot,
contents,
kind,
CreatePipProvenance(usage),
ToStringIds(tags),
ToStringIds(patterns),
scrub);
if (PipGraph != null)
{
sealedDirectory = PipGraph.AddSealDirectory(pip, GetValuePipId());
if (!sealedDirectory.IsValid)
{
return(false);
}
}
else
{
sealedDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(directoryRoot);
}
return(true);
}
/// <summary>
/// Augments the processBuilder with the OS dependencies.
/// </summary>
/// <param name="processBuilder">builder to use</param>
/// <param name="untrackInsteadSourceSeal">when true, directories that are meant to be source sealed are untracked instead</param>
public bool ProcessDefaults(ProcessBuilder processBuilder, bool untrackInsteadSourceSeal = false)
{
if (processBuilder.Options.HasFlag(Process.Options.DependsOnCurrentOs))
{
// process source seal directories: either source seal them or untrack them, depending on 'untrackInsteadSourceSeal'
if (untrackInsteadSourceSeal)
{
foreach (var sourceSealDirPath in m_sourceSealDirectoryPaths)
{
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(sourceSealDirPath));
}
}
else
{
var defaultSourceSealDirs = m_lazySourceSealDirectories.Value;
if (!defaultSourceSealDirs.IsValid)
{
return(false);
}
foreach (var inputDirectory in defaultSourceSealDirs.Directories)
{
processBuilder.AddInputDirectory(inputDirectory);
}
}
// add untracked files
foreach (var untrackedFile in m_untrackedFiles)
{
processBuilder.AddUntrackedFile(untrackedFile);
}
// add untracked directories
foreach (var untrackedDirectory in m_untrackedDirectories)
{
processBuilder.AddUntrackedDirectoryScope(untrackedDirectory);
}
}
return(true);
}
public void ReadingUnderAnExclusiveOpaqueIsBlocked()
{
var exclusiveOpaqueRoot = Path.Combine(ObjectRoot, "exclusiveOpaque");
var source = CreateSourceFile(exclusiveOpaqueRoot);
// Read the source file
var reader = ScheduleProcessWithUndeclaredReads(source);
// Create an exclusive opaque that produces the same file
var builder = CreatePipBuilder(new List <Operation>()
{
Operation.WriteFile(source, doNotInfer: true)
});
var exclusiveOpaqueDirectoryArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(Context.PathTable, exclusiveOpaqueRoot);
builder.AddOutputDirectory(exclusiveOpaqueDirectoryArtifact, SealDirectoryKind.Opaque);
var writer = SchedulePipBuilder(builder);
// A violation should be detected when reading the produced output as a source file
// Force an execution order to avoid write locks
RunScheduler(constraintExecutionOrder: new[] { ((Pip)reader.Process, (Pip)writer.Process) }).AssertFailure();
public void ExecutionUntrackTempFolder()
{
AbsolutePath tempDirectory = CreateUniqueDirectory(ObjectRoot);
FileArtifact tempFile = CreateOutputFileArtifact(tempDirectory);
ProcessBuilder builder = CreatePipBuilder(new[]
{
Operation.ReadFile(CreateSourceFile()),
Operation.WriteFile(CreateOutputFileArtifact()),
Operation.WriteFile(tempFile, doNotInfer: true),
Operation.ReadFile(tempFile, doNotInfer: true)
});
builder.Options |= Process.Options.RequiresAdmin;
builder.SetTempDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(tempDirectory));
ProcessWithOutputs process = SchedulePipBuilder(builder);
RunScheduler().AssertSuccess();
RunScheduler().AssertCacheHit(process.Process.PipId);
}
/// <summary>
/// Creates a directory artifact which may be queried with <see cref="ListSealedDirectoryContents"/>,
/// and whose members may be queried with <see cref="TryQuerySealedInputContent"/>.
/// Each mentioned path must have been added explicitly with <see cref="AddAbsentPath"/> or <see cref="AddFile"/>
/// </summary>
public DirectoryArtifact SealDir(string root, params string[] contents)
{
var rootPath = Path(root);
FileArtifact[] artifacts = new FileArtifact[contents.Length];
for (int i = 0; i < contents.Length; i++)
{
var path = Path(contents[i]);
if (!path.IsWithin(Context.PathTable, rootPath))
{
XAssert.Fail("Root {0} does not contain {1}", root, contents[i]);
}
artifacts[i] = FileArtifact.CreateSourceFile(path);
}
DirectoryArtifact newArtifact = DirectoryArtifact.CreateDirectoryArtifactForTesting(rootPath, m_nextDirectorySealId++);
m_env.SetSealedDirectoryContents(newArtifact, artifacts);
return(newArtifact);
}
internal static bool TryDeserializeDirectoryArtifact(string input, out DirectoryArtifact directoryArtifact)
{
var components = input.Split(':');
if (components.Length != 3)
{
directoryArtifact = default;
return(false);
}
if (!int.TryParse(components[0], out var pathId) ||
!uint.TryParse(components[1], out var partialSealId) ||
!int.TryParse(components[2], out var isSharedOpaque))
{
directoryArtifact = default;
return(false);
}
directoryArtifact = new DirectoryArtifact(new AbsolutePath(pathId), partialSealId, isSharedOpaque == 1);
return(true);
}
/// <param name="topOnly">When true, the SourceSealedDirectory will be a topOnly. When false, it will be a recursive SourceSealedDirectory</param>
/// <param name="probeOnly">When true, the source file will be probed without reading</param>
/// <param name="alsoDeclareAsInput">When true, the file accessed under the SourceSealDirectory will also be declared as an input</param>
public void ValidateCachingBehavior(bool topOnly, bool probeOnly, bool alsoDeclareAsInput = false)
{
// Create a very simple graph with a process that consumes files from a source sealed directory
var source1 = topOnly ? CreateSourceFile(SourceRoot) : CreateSourceFile(Path.Combine(SourceRoot, "nested"));
WriteSourceFile(source1);
// Create a graph with a SealedSourceDirectory
DirectoryArtifact dir = SealDirectory(SourceRootPath, topOnly ? SealDirectoryKind.SourceTopDirectoryOnly : SealDirectoryKind.SourceAllDirectories);
// Set up test process
var builder = CreatePipBuilder(new Operation[]
{
Operation.WriteFile(CreateOutputFileArtifact(ObjectRoot)),
// don't take dependencies on files in SealedSourceDirectory
probeOnly ? Operation.Probe(source1, doNotInfer: !alsoDeclareAsInput) : Operation.ReadFile(source1, doNotInfer: !alsoDeclareAsInput)
});
builder.AddInputDirectory(dir);
Process process = SchedulePipBuilder(builder).Process;
// Perform builds:
RunScheduler().AssertCacheMiss(process.PipId);
RunScheduler().AssertCacheHit(process.PipId);
// Modify an input file and make sure there's a cache miss
WriteSourceFile(source1);
if (probeOnly && !alsoDeclareAsInput)
{
// Files probes that are not declared as input do not cause cache misses when the content changes.
RunScheduler().AssertCacheHit(process.PipId);
}
else
{
RunScheduler().AssertCacheMiss(process.PipId);
}
RunScheduler().AssertCacheHit(process.PipId);
}
private EvaluationResult ComposeSharedOpaqueDirectories(Context context, ModuleLiteral env, EvaluationStackFrame args)
{
AbsolutePath root;
ArrayLiteral contents;
SealDirectoryContentFilter?contentFilter;
if (args.Length > 0 && args[0].Value is ObjectLiteral)
{
var obj = Args.AsObjectLiteral(args, 0);
var directory = Converter.ExtractDirectory(obj, m_sealRoot, allowUndefined: false);
root = directory.Path;
contents = Converter.ExtractArrayLiteral(obj, m_sealDirectories, allowUndefined: false);
var filterObj = Converter.ExtractObjectLiteral(obj, m_sealDirectoryContentFilter, allowUndefined: true);
contentFilter = GetContentFilterHelper(filterObj);
}
else
{
root = Args.AsPath(args, 0, false);
contents = Args.AsArrayLiteral(args, 1);
var filterObj = Args.AsObjectLiteralOptional(args, 2);
contentFilter = GetContentFilterHelper(filterObj);
}
var directories = new DirectoryArtifact[contents.Length];
for (int i = 0; i < contents.Length; ++i)
{
directories[i] = Converter.ExpectSharedOpaqueDirectory(contents[i], context: new ConversionContext(pos: i, objectCtx: contents)).Root;
}
if (!context.GetPipConstructionHelper().TryComposeSharedOpaqueDirectory(root, directories, SealDirectoryCompositionActionKind.WidenDirectoryCone, contentFilter, description: null, tags: null, out var compositeSharedOpaque))
{
// Error should have been logged
return(EvaluationResult.Error);
}
var result = new StaticDirectory(compositeSharedOpaque, SealDirectoryKind.SharedOpaque, s_emptySealContents.WithCompatibleComparer(OrdinalPathOnlyFileArtifactComparer.Instance));
return(new EvaluationResult(result));
}
/// <nodoc/>
public bool TryComposeSharedOpaqueDirectory(
AbsolutePath directoryRoot,
IReadOnlyList <DirectoryArtifact> contents,
[CanBeNull] string description,
string[] tags,
out DirectoryArtifact sharedOpaqueDirectory)
{
Contract.Requires(directoryRoot.IsValid);
Contract.Requires(contents != null);
if (PipGraph == null)
{
sharedOpaqueDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(directoryRoot);
return(true);
}
PipData usage = PipDataBuilder.CreatePipData(Context.StringTable, string.Empty, PipDataFragmentEscaping.NoEscaping, description != null
? new PipDataAtom[] { description }
: new PipDataAtom[] { "'", directoryRoot, " [", contents.Count.ToString(CultureInfo.InvariantCulture), " shared opaque directories]" });
sharedOpaqueDirectory = PipGraph.ReserveSharedOpaqueDirectory(directoryRoot);
var pip = new CompositeSharedOpaqueSealDirectory(
directoryRoot,
contents,
CreatePipProvenance(usage),
ToStringIds(tags));
// The seal directory is ready to be initialized, since the directory artifact has been reserved already
pip.SetDirectoryArtifact(sharedOpaqueDirectory);
sharedOpaqueDirectory = PipGraph.AddSealDirectory(pip, GetValuePipId());
if (!sharedOpaqueDirectory.IsValid)
{
return(false);
}
return(true);
}
/// <inheritdoc />
protected override void AddWeakFingerprint(IFingerprinter fingerprinter, SealDirectory sealDirectory)
{
Contract.Requires(fingerprinter != null);
Contract.Requires(sealDirectory != null);
base.AddWeakFingerprint(fingerprinter, sealDirectory);
// For non-composite shared opaque directories, contents and composed directories are always empty, and therefore the static fingerprint
// is not strong enough, i.e. multiple shared opaques can share the same directory root. So in this case we need to add the fingerprint of the producer
if (sealDirectory.Kind == SealDirectoryKind.SharedOpaque && !sealDirectory.IsComposite)
{
DirectoryArtifact directory = sealDirectory.Directory;
fingerprinter.Add(directory, m_directoryProducerFingerprintLookup(directory).Hash);
}
if (!ExcludeSemiStableHashOnFingerprintingSealDirectory && !sealDirectory.Kind.IsDynamicKind())
{
// A statically sealed directory can exist as multiple different instances, e.g., one can have partially sealed directories with the same root and member set.
// To distinguish those instances, we include the semi stable hash as part of the static fingerprint.
fingerprinter.Add("SemiStableHash", sealDirectory.SemiStableHash);
}
}
public void ScheduleFullSealDirectoryWithOpaqueAsContentBehavior(bool includeOpaqueAsContent)
{
using (TestEnv env = TestEnv.CreateTestEnvWithPausedScheduler())
{
AbsolutePath directoryPath = env.Paths.CreateAbsolutePath(env.ObjectRoot, "seal");
AbsolutePath sod = env.Paths.CreateAbsolutePath(directoryPath, "sod");
// A pip that produces an output directory under the seal root
var pip = CreatePipBuilderWithTag(env, "test");
pip.AddOutputDirectory(sod, SealDirectoryKind.SharedOpaque);
var pipResult = env.PipConstructionHelper.AddProcess(pip);
var sharedOpaqueArtifact = pipResult.GetOpaqueDirectory(sod);
FileArtifact a = ScheduleWriteOutputFileUnderDirectory(env, directoryPath, "a");
// We should be able to schedule a seal directory with the produced file and optionally the opaque as content
DirectoryArtifact sealDirectory = ScheduleSealDirectory(
env,
directoryPath,
new[] { a },
includeOpaqueAsContent? new[] { sharedOpaqueArtifact } : new DirectoryArtifact[] { });
var graph = env.PipGraph.Build();
if (includeOpaqueAsContent)
{
// Everything should be good in this case
Assert.NotNull(graph);
// The fully seal directory should have a dependency on the opaque
Assert.True(graph.DataflowGraph.IsReachableFrom(graph.GetSealedDirectoryNode(sharedOpaqueArtifact), graph.GetSealedDirectoryNode(sealDirectory)));
}
else
{
// We should fail at scheduling the graph because of a missing output directory
Assert.Null(graph);
AssertErrorEventLogged(LogEventId.InvalidGraphSinceFullySealedDirectoryIncompleteDueToMissingOutputDirectories);
}
}
}
private static EvaluationResult GetPathValues(Context context, EvaluationStackFrame args, Type type)
{
var name = Args.AsString(args, 0);
var separator = Args.AsString(args, 1);
string strValue = GetRawValue(context, name);
var entry = context.TopStack;
if (string.IsNullOrWhiteSpace(strValue))
{
return(EvaluationResult.Create(ArrayLiteral.CreateWithoutCopy(CollectionUtilities.EmptyArray <EvaluationResult>(), entry.InvocationLocation, entry.Path)));
}
var values = separator.Length == 0 ? new[] { strValue } : strValue.Split(new[] { separator }, StringSplitOptions.RemoveEmptyEntries);
var pathsOrFiles = new List <EvaluationResult>();
for (int i = 0; i < values.Length; ++i)
{
if (!string.IsNullOrWhiteSpace(values[i]))
{
AbsolutePath path = ParsePath(context, name, values[i], 1);
EvaluationResult result =
type == typeof(AbsolutePath) ? EvaluationResult.Create(path) :
type == typeof(FileArtifact) ? EvaluationResult.Create(FileArtifact.CreateSourceFile(path)) :
type == typeof(DirectoryArtifact) ? EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(path)) :
EvaluationResult.Undefined;
if (result.IsUndefined)
{
throw Contract.AssertFailure(I($"Cannot convert paths to typeof({type.Name})"));
}
pathsOrFiles.Add(result);
}
}
return(EvaluationResult.Create(ArrayLiteral.CreateWithoutCopy(pathsOrFiles.ToArray(), entry.InvocationLocation, entry.Path)));
}
private EvaluationResult[] EnumerateFilesOrDirectories(
Context context,
string directoryPath,
string searchPattern,
bool isRecursive,
bool enumerateDirectory)
{
var fileSystem = context.FrontEndContext.FileSystem;
if (enumerateDirectory)
{
return(fileSystem
.EnumerateDirectories(AbsolutePath.Create(context.PathTable, directoryPath), searchPattern, isRecursive)
.Select(ap => EvaluationResult.Create(DirectoryArtifact.CreateWithZeroPartialSealId(ap)))
.ToArray());
}
return(fileSystem
.EnumerateFiles(AbsolutePath.Create(context.PathTable, directoryPath), searchPattern, isRecursive)
.Select(ap => EvaluationResult.Create(FileArtifact.CreateSourceFile(ap)))
.ToArray());
}
private void AddAdditionalOutputDirectories(ProcessBuilder processBuilder, AbsolutePath projectFolder)
{
if (m_resolverSettings.AdditionalOutputDirectories == null)
{
return;
}
foreach (DiscriminatingUnion <AbsolutePath, RelativePath> directoryUnion in m_resolverSettings.AdditionalOutputDirectories)
{
object directory = directoryUnion.GetValue();
if (directory is AbsolutePath absolutePath)
{
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(absolutePath), SealDirectoryKind.SharedOpaque);
}
else
{
// The specified relative path is interpreted relative to the project directory folder
AbsolutePath absoluteDirectory = projectFolder.Combine(PathTable, (RelativePath)directory);
processBuilder.AddOutputDirectory(DirectoryArtifact.CreateWithZeroPartialSealId(absoluteDirectory), SealDirectoryKind.SharedOpaque);
}
}
}
public void TranslateGlobalUntrackedScope(bool translate, Process.Options requireGlobalDependencies)
{
DirectoryArtifact sourceDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(SourceRoot, prefix: "sourceDir"));
DirectoryArtifact targetDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(SourceRoot, prefix: "targetDir"));
FileArtifact outputFileInTargetDir = CreateOutputFileArtifact(ArtifactToString(targetDirectory));
FileArtifact inputFileInTargetDir = CreateSourceFile(ArtifactToString(targetDirectory));
Configuration.Sandbox.GlobalUnsafeUntrackedScopes.Add(sourceDirectory);
if (translate)
{
DirectoryTranslator = new DirectoryTranslator();
DirectoryTranslator.AddTranslation(ArtifactToString(sourceDirectory), ArtifactToString(targetDirectory));
}
var ops = new Operation[]
{
Operation.ReadFile(inputFileInTargetDir, doNotInfer: true),
Operation.WriteFile(outputFileInTargetDir)
};
var builder = CreatePipBuilder(ops);
builder.Options |= requireGlobalDependencies;
Process pip = SchedulePipBuilder(builder).Process;
if (translate && ((requireGlobalDependencies & Process.Options.RequireGlobalDependencies) == Process.Options.RequireGlobalDependencies))
{
RunScheduler().AssertCacheMiss(pip.PipId);
RunScheduler().AssertCacheHit(pip.PipId);
}
else
{
RunScheduler().AssertFailure();
AssertWarningEventLogged(EventId.ProcessNotStoredToCacheDueToFileMonitoringViolations);
AssertErrorEventLogged(EventId.FileMonitoringError);
}
}
public void IncrementalPreserveOutputTool()
{
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.PreserveOutputs = PreserveOutputsMode.Enabled;
Configuration.IncrementalTools = new List <RelativePath>
{
RelativePath.Create(Context.StringTable, TestProcessToolName)
};
AbsolutePath readonlyRootPath;
AbsolutePath.TryCreate(Context.PathTable, ReadonlyRoot, out readonlyRootPath);
// Create /readonly/a.txt
FileArtifact aTxtFile = CreateFileArtifactWithName("a.txt", ReadonlyRoot);
WriteSourceFile(aTxtFile);
DirectoryArtifact readonlyRootDir = SealDirectory(readonlyRootPath, SealDirectoryKind.SourceAllDirectories);
var builder = CreatePipBuilder(new Operation[]
{
Operation.Probe(aTxtFile, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact())
});
builder.AddInputDirectory(readonlyRootDir);
builder.Options |= Process.Options.AllowPreserveOutputs;
builder.Options |= Process.Options.IncrementalTool;
var pip = SchedulePipBuilder(builder).Process;
RunScheduler().AssertCacheMiss(pip.PipId);
RunScheduler().AssertCacheHit(pip.PipId);
WriteSourceFile(aTxtFile);
RunScheduler().AssertCacheMiss(pip.PipId);
}
private ObjectInfo DirectoryArtifactInfo(DirectoryArtifact d)
{
if (!d.IsValid)
{
return(new ObjectInfo("Invalid"));
}
var name = d.Path.GetName(PathTable).ToString(StringTable);
var kind = d.IsSharedOpaque ? "shared opaque" : d.IsOutputDirectory() ? "exclusive opaque" : "source";
return(new ObjectInfo(
preview: $"{name} [{kind}]",
properties: new[]
{
new Property("Path", d.Path.ToString(PathTable)),
new Property("PartialSealId", d.PartialSealId),
new Property("Kind", kind),
d.IsOutputDirectory() ? new Property("Producer", () => PipGraph.GetProducer(d)) : null,
new Property("Consumers", PipGraph.GetConsumingPips(d.Path).ToArray()),
d.PartialSealId > 0 ? new Property("Members", () => PipGraph.ListSealedDirectoryContents(d)) : null
}
.Where(p => p != null)));
}
public void SetSealedSourceDirectory(DirectoryArtifact directory, bool allDirectories, params AbsolutePath[] dynamicallyAccessedPathInDirectory)
{
var directoryPath = directory.Path;
var sealedDiretoriesToUse = allDirectories ? m_knownSealedSourceDirectoriesAllDirectories : m_knownSealedSourceDirectoriesTopDirectoryOnly;
sealedDiretoriesToUse.Add(directoryPath);
foreach (AbsolutePath path in dynamicallyAccessedPathInDirectory)
{
if (allDirectories)
{
Contract.Assume(path.IsWithin(Context.PathTable, directoryPath));
}
else
{
Contract.Assume(
path.GetParent(Context.PathTable) == directoryPath,
"If not recursive all files must be directly under the folder.");
}
m_knownSealedArtifacts[path] = FileArtifact.CreateSourceFile(path);
}
}
private void SetUntrackedFilesAndDirectories(ProcessBuilder processBuilder)
{
// On some machines, the current user and public user desktop.ini are read by Powershell.exe.
// Ignore accesses to the user profile and Public common user profile.
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile)));
if (Engine.TryGetBuildParameter("PUBLIC", m_frontEndName, out string publicDir))
{
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(AbsolutePath.Create(PathTable, publicDir)));
}
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
// Git accesses should be ignored if .git directory is there
var gitDirectory = Root.Combine(PathTable, ".git");
if (Engine.DirectoryExists(gitDirectory))
{
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(gitDirectory));
processBuilder.AddUntrackedFile(FileArtifact.CreateSourceFile(Root.Combine(PathTable, ".gitattributes")));
processBuilder.AddUntrackedFile(FileArtifact.CreateSourceFile(Root.Combine(PathTable, ".gitignore")));
}
}
public void CreatedDirectoriesUnderSharedOpaquesAreNotPartOfTheFingerprint()
{
string dir = Path.Combine(SourceRoot, "dir");
AbsolutePath dirPath = AbsolutePath.Create(Context.PathTable, dir);
DirectoryArtifact dirToEnumerate = DirectoryArtifact.CreateWithZeroPartialSealId(dirPath);
Configuration.Logging.CacheMissAnalysisOption = CacheMissAnalysisOption.LocalMode();
AbsolutePath nestedDirPath = dirPath.Combine(Context.PathTable, "nested");
var nestedDir = DirectoryArtifact.CreateWithZeroPartialSealId(nestedDirPath);
var operations = new List <Operation>
{
// Create a directory nested into the one that is going to be enumerated
Operation.CreateDir(nestedDir, doNotInfer: true),
Operation.EnumerateDir(dirToEnumerate, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()) // dummy output
};
var builder = CreatePipBuilder(operations);
// This makes sure we use the right file system, which is aware of alien files
builder.Options |= global::BuildXL.Pips.Operations.Process.Options.AllowUndeclaredSourceReads;
// Define the shared opaque
builder.AddOutputDirectory(dirPath, global::BuildXL.Pips.Operations.SealDirectoryKind.SharedOpaque);
var pip = SchedulePipBuilder(builder);
// Run once
RunScheduler().AssertSuccess();
// Simulate shared opaque scrubbing
FileUtilities.DeleteDirectoryContents(nestedDirPath.ToString(Context.PathTable), deleteRootDirectory: true);
// This should be a cache hit. Directories created by a pip are not part of the fingerprint
RunScheduler().AssertCacheHit(pip.Process.PipId);
}
public void ValidateCachingBehaviorUntrackedMount(FileArtifact file, DirectoryArtifact dir, bool undefinedMount)
{
// Dynamically observed accesses on the file and directory
Process pip = CreateAndSchedulePipBuilder(new Operation[]
{
Operation.ReadFile(file),
Operation.EnumerateDir(dir),
Operation.WriteFile(CreateOutputFileArtifact())
}).Process;
RunScheduler().AssertCacheMiss(pip.PipId);
RunScheduler().AssertCacheHit(pip.PipId);
// Create previously absent /file
File.WriteAllText(ArtifactToString(file), "hello");
RunScheduler().AssertCacheHit(pip.PipId);
// Create previously absent /dir
Directory.CreateDirectory(ArtifactToString(dir));
if (undefinedMount)
{
//// TODO: Bug #1087986 - This is a miss for undefined mounts
RunScheduler().AssertCacheMiss(pip.PipId);
}
RunScheduler().AssertCacheHit(pip.PipId);
// Modify /file
File.WriteAllText(ArtifactToString(file), "world");
RunScheduler().AssertCacheHit(pip.PipId);
// Modify /dir by creating /dir/undefinedNestedFile
FileArtifact undefinedNestedFile = new FileArtifact(CreateUniqueSourcePath(SourceRootPrefix, ArtifactToString(dir)));
File.Create(ArtifactToString(undefinedNestedFile));
RunScheduler().AssertCacheHit(pip.PipId);
}
public void EnumeratingDirectoryUnderSharedOpaqueUsesMinimalGraphFileSystem()
{
// Enable minimal lazy output materialization to ensure files are not placed on disk after cache hit
Configuration.Schedule.RequiredOutputMaterialization = RequiredOutputMaterialization.Minimal;
// Enumeration path is sub path of shared opaque to ensure enumerating pip doesn't explicitly
// mention the path in its dependencies
var enumerationPath = Path.Combine(SharedOpaqueDirectoryRoot, "enum");
// Create a pip that writes a file under a shared opaque
var fileToWrite = CreateOutputFileArtifact(enumerationPath);
var sharedOpaqueProducer = CreateAndScheduleSharedOpaqueProducer(enumerationPath, filesToProduce: fileToWrite);
// Create a pip which enumerates the shared opaque
var enumeratingPipBuilder = CreatePipBuilder(new[]
{
Operation.EnumerateDir(DirectoryArtifact.CreateWithZeroPartialSealId(Context.PathTable, enumerationPath), doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()) // dummy output
});
// Mark the pip with allow undeclared source reads to test regression of directory enumeration behavior in this case
enumeratingPipBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
enumeratingPipBuilder.AddInputDirectory(sharedOpaqueProducer.ProcessOutputs.GetOutputDirectories().Single().Root);
var enumeratingPip = SchedulePipBuilder(enumeratingPipBuilder);
RunScheduler().AssertSuccess();
// After deleting the enumerated directory contents we should still get a hit even though the
// state of the file system has changed since we are using the minimal graph file system
FileUtilities.DeleteDirectoryContents(enumerationPath);
RunScheduler().AssertCacheHit(enumeratingPip.Process.PipId, sharedOpaqueProducer.Process.PipId);
AssertTrue(Directory.GetFiles(enumerationPath, "*.*", SearchOption.AllDirectories).Length == 0, "Enumerated directory should be empty due to lazy materialization");
}
public void AddRealPath(string fullPath, PathExistence existence)
{
if (existence == PathExistence.Nonexistent)
{
return;
}
var path = AbsolutePath.Create(m_pathTable, fullPath);
FileOrDirectoryArtifact member = FileOrDirectoryArtifact.Invalid;
while (path.IsValid)
{
if (existence == PathExistence.ExistsAsFile)
{
m_files.Add(path);
member = new FileArtifact(path);
}
else
{
var members = m_directories.GetOrAdd(path, p => new HashSet <FileOrDirectoryArtifact>());
if (member.IsValid)
{
if (!members.Add(member))
{
return;
}
}
member = DirectoryArtifact.CreateWithZeroPartialSealId(path);
}
existence = PathExistence.ExistsAsDirectory;
path = path.GetParent(m_pathTable);
}
}
