public void DoubleWritePolicyDeterminesViolationSeverity(DoubleWritePolicy doubleWritePolicy)
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(
LoggingContext,
context,
graph,
// Set this to test the logic of base.HandleDependencyViolation(...) instead of the overriding fake
doLogging: true,
collectNonErrorViolations: true);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
AbsolutePath producerOutput = CreateAbsolutePath(context, DoubleWritePath);
Process producer = graph.AddProcess(producerOutput, doubleWritePolicy);
Process violator = graph.AddProcess(violatorOutput, doubleWritePolicy);
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.ReadWrite, producerOutput) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null,
ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .Empty,
out _);
analyzer.AssertContainsViolation(
new DependencyViolation(
FileMonitoringViolationAnalyzer.DependencyViolationType.DoubleWrite,
FileMonitoringViolationAnalyzer.AccessLevel.Write,
producerOutput,
violator,
producer),
"The violator is after the producer, so this should be a double-write on the produced path.");
analyzer.AssertNoExtraViolationsCollected();
AssertVerboseEventLogged(LogEventId.DependencyViolationDoubleWrite);
// Based on the double write policy, the violation is an error or a warning
if (doubleWritePolicy == DoubleWritePolicy.DoubleWritesAreErrors)
{
AssertErrorEventLogged(EventId.FileMonitoringError);
}
else
{
AssertWarningEventLogged(EventId.FileMonitoringWarning);
}
}
/// <summary>
/// Whether the double-write policy implies that double writes should be flagged as errors
/// </summary>
public static bool ImpliesDoubleWriteIsError(this DoubleWritePolicy policy)
{
switch (policy)
{
case DoubleWritePolicy.DoubleWritesAreErrors:
return(true);
case DoubleWritePolicy.UnsafeFirstDoubleWriteWins:
return(false);
default:
throw new InvalidOperationException("Unexpected double write policy " + policy.ToString());
}
}
public void DoubleWritePolicyIsContentAware(DoubleWritePolicy doubleWritePolicy)
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(
LoggingContext,
context,
graph,
// Set this to test the logic of base.HandleDependencyViolation(...) instead of the overriding fake
doLogging: true,
collectNonErrorViolations: true);
// Create the path where the double write will occur, and a random file content that will be used for both producers
AbsolutePath doubleWriteOutput = CreateAbsolutePath(context, JunkPath);
ContentHash contentHash = ContentHashingUtilities.CreateRandom();
var fileContentInfo = new FileContentInfo(contentHash, contentHash.Length);
var outputsContent = new (FileArtifact, FileMaterializationInfo, PipOutputOrigin)[]
private void ScheduleDoubleWriteProducers(
AbsolutePath sharedOpaqueDirPath,
FileArtifact doubleWriteArtifact,
ContainerIsolationLevel containerIsolationLevel,
DoubleWritePolicy doubleWritePolicy,
out ProcessWithOutputs firstProducer,
out ProcessWithOutputs secondProducer)
{
var firstProducerBuilder = CreateFileInSharedOpaqueBuilder(containerIsolationLevel, doubleWritePolicy, doubleWriteArtifact, "first", sharedOpaqueDirPath);
firstProducer = SchedulePipBuilder(firstProducerBuilder);
var secondProducerBuilder = CreateFileInSharedOpaqueBuilder(containerIsolationLevel, doubleWritePolicy, doubleWriteArtifact, "second", sharedOpaqueDirPath);
// Let's order this so who is the violator is deterministic
secondProducerBuilder.AddInputDirectory(firstProducer.Process.DirectoryOutputs.First());
secondProducer = SchedulePipBuilder(secondProducerBuilder);
}
/// <summary>
/// Adds a fake process pip that produces only the given path.
/// </summary>
public Process AddProcess(AbsolutePath producedPath, DoubleWritePolicy doubleWritePolicy = DoubleWritePolicy.DoubleWritesAreErrors)
{
Contract.Assume(!m_pathProducers.ContainsKey(producedPath), "Each path may have only one producer (no rewrites)");
AbsolutePath workingDirectory = AbsolutePath.Create(m_context.PathTable, PathGeneratorUtilities.GetAbsolutePath("X", ""));
AbsolutePath exe = AbsolutePath.Create(m_context.PathTable, PathGeneratorUtilities.GetAbsolutePath("X", "fake.exe"));
var process = new Process(
executable: FileArtifact.CreateSourceFile(exe),
workingDirectory: workingDirectory,
arguments: PipDataBuilder.CreatePipData(m_context.StringTable, string.Empty, PipDataFragmentEscaping.NoEscaping),
responseFile: FileArtifact.Invalid,
responseFileData: PipData.Invalid,
environmentVariables: ReadOnlyArray <EnvironmentVariable> .Empty,
standardInput: FileArtifact.Invalid,
standardOutput: FileArtifact.Invalid,
standardError: FileArtifact.Invalid,
standardDirectory: workingDirectory,
warningTimeout: null,
timeout: null,
dependencies: ReadOnlyArray <FileArtifact> .FromWithoutCopy(FileArtifact.CreateSourceFile(exe)),
outputs: ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(FileArtifact.CreateSourceFile(producedPath).CreateNextWrittenVersion().WithAttributes()),
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty,
directoryOutputs: ReadOnlyArray <DirectoryArtifact> .Empty,
orderDependencies: ReadOnlyArray <PipId> .Empty,
untrackedPaths: ReadOnlyArray <AbsolutePath> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .Empty,
tags: ReadOnlyArray <StringId> .Empty,
successExitCodes: ReadOnlyArray <int> .Empty,
semaphores: ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(m_context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty,
doubleWritePolicy: doubleWritePolicy);
process.PipId = AllocateNextPipId();
m_pips.Add(process.PipId, process);
m_pathProducers.Add(producedPath, process);
return(process);
}
/// <summary>
/// Clone and override select properties.
/// </summary>
public Process Override(
FileArtifact?executable = null,
AbsolutePath?workingDirectory = null,
PipData?arguments = null,
FileArtifact?responseFile = null,
PipData?responseFileData = null,
ReadOnlyArray <EnvironmentVariable>?environmentVariables = null,
StandardInput?standardInput = null,
FileArtifact?standardOutput = null,
FileArtifact?standardError = null,
AbsolutePath?standardDirectory = null,
TimeSpan?warningTimeout = null,
TimeSpan?timeout = null,
ReadOnlyArray <FileArtifact>?dependencies = null,
ReadOnlyArray <FileArtifactWithAttributes>?fileOutputs = null,
ReadOnlyArray <DirectoryArtifact>?directoryDependencies = null,
ReadOnlyArray <DirectoryArtifact>?directoryOutputs = null,
ReadOnlyArray <PipId>?orderDependencies = null,
ReadOnlyArray <AbsolutePath>?untrackedPaths = null,
ReadOnlyArray <AbsolutePath>?untrackedScopes = null,
ReadOnlyArray <StringId>?tags = null,
ReadOnlyArray <int>?successExitCodes = null,
ReadOnlyArray <ProcessSemaphoreInfo>?semaphores = null,
PipProvenance provenance = null,
StringId?toolDescription = null,
ReadOnlyArray <AbsolutePath>?additionalTempDirectories = null,
RegexDescriptor?warningRegex = null,
RegexDescriptor?errorRegex = null,
AbsolutePath?uniqueOutputDirectory = null,
AbsolutePath?redirectedDirectoryRoot = null,
AbsolutePath?tempDirectory = null,
Options?options = null,
bool?testRetries = null,
ServiceInfo serviceInfo = null,
ReadOnlyArray <int>?retryExitCodes = null,
ReadOnlyArray <PathAtom>?allowedSurvivingChildProcessNames = null,
TimeSpan?nestedProcessTerminationTimeout = null,
AbsentPathProbeInUndeclaredOpaquesMode absentPathProbeMode = AbsentPathProbeInUndeclaredOpaquesMode.Unsafe,
DoubleWritePolicy doubleWritePolicy = DoubleWritePolicy.DoubleWritesAreErrors,
ContainerIsolationLevel containerIsolationLevel = ContainerIsolationLevel.None)
{
return(new Process(
executable ?? Executable,
workingDirectory ?? WorkingDirectory,
arguments ?? Arguments,
responseFile ?? ResponseFile,
responseFileData ?? ResponseFileData,
environmentVariables ?? EnvironmentVariables,
standardInput ?? StandardInput,
standardOutput ?? StandardOutput,
standardError ?? StandardError,
standardDirectory ?? StandardDirectory,
warningTimeout ?? WarningTimeout,
timeout ?? Timeout,
dependencies ?? Dependencies,
fileOutputs ?? FileOutputs,
directoryDependencies ?? DirectoryDependencies,
directoryOutputs ?? DirectoryOutputs,
orderDependencies ?? OrderDependencies,
untrackedPaths ?? UntrackedPaths,
untrackedScopes ?? UntrackedScopes,
tags ?? Tags,
successExitCodes ?? SuccessExitCodes,
semaphores ?? Semaphores,
provenance ?? Provenance,
toolDescription ?? ToolDescription,
additionalTempDirectories ?? AdditionalTempDirectories,
warningRegex ?? WarningRegex,
errorRegex ?? ErrorRegex,
uniqueOutputDirectory ?? UniqueOutputDirectory,
redirectedDirectoryRoot ?? UniqueRedirectedDirectoryRoot,
tempDirectory ?? TempDirectory,
options ?? ProcessOptions,
testRetries ?? TestRetries,
serviceInfo ?? ServiceInfo,
retryExitCodes ?? RetryExitCodes,
allowedSurvivingChildProcessNames,
nestedProcessTerminationTimeout,
absentPathProbeMode,
doubleWritePolicy,
containerIsolationLevel));
}
/// <summary>
/// Class constructor
/// </summary>
public Process(
FileArtifact executable,
AbsolutePath workingDirectory,
PipData arguments,
FileArtifact responseFile,
PipData responseFileData,
ReadOnlyArray <EnvironmentVariable> environmentVariables,
StandardInput standardInput,
FileArtifact standardOutput,
FileArtifact standardError,
AbsolutePath standardDirectory,
TimeSpan?warningTimeout,
TimeSpan?timeout,
ReadOnlyArray <FileArtifact> dependencies,
ReadOnlyArray <FileArtifactWithAttributes> outputs,
ReadOnlyArray <DirectoryArtifact> directoryDependencies,
ReadOnlyArray <DirectoryArtifact> directoryOutputs,
ReadOnlyArray <PipId> orderDependencies,
ReadOnlyArray <AbsolutePath> untrackedPaths,
ReadOnlyArray <AbsolutePath> untrackedScopes,
ReadOnlyArray <StringId> tags,
ReadOnlyArray <int> successExitCodes,
ReadOnlyArray <ProcessSemaphoreInfo> semaphores,
PipProvenance provenance,
StringId toolDescription,
ReadOnlyArray <AbsolutePath> additionalTempDirectories,
RegexDescriptor warningRegex = default,
RegexDescriptor errorRegex = default,
AbsolutePath uniqueOutputDirectory = default,
AbsolutePath uniqueRedirectedDirectoryRoot = default,
AbsolutePath tempDirectory = default,
Options options = default,
bool testRetries = false,
ServiceInfo serviceInfo = null,
ReadOnlyArray <int>?retryExitCodes = null,
ReadOnlyArray <PathAtom>?allowedSurvivingChildProcessNames = null,
TimeSpan?nestedProcessTerminationTimeout = null,
AbsentPathProbeInUndeclaredOpaquesMode absentPathProbeMode = AbsentPathProbeInUndeclaredOpaquesMode.Unsafe,
DoubleWritePolicy doubleWritePolicy = DoubleWritePolicy.DoubleWritesAreErrors,
ContainerIsolationLevel containerIsolationLevel = ContainerIsolationLevel.None)
{
Contract.Requires(executable.IsValid);
Contract.Requires(workingDirectory.IsValid);
Contract.Requires(arguments.IsValid);
Contract.RequiresForAll(environmentVariables, environmentVariable => environmentVariable.Name.IsValid);
Contract.RequiresForAll(environmentVariables, environmentVariable => environmentVariable.Value.IsValid ^ environmentVariable.IsPassThrough);
Contract.Requires(dependencies.IsValid);
Contract.RequiresForAll(dependencies, dependency => dependency.IsValid);
Contract.Requires(directoryDependencies.IsValid);
Contract.RequiresForAll(directoryDependencies, directoryDependency => directoryDependency.IsValid);
Contract.Requires(outputs.IsValid);
Contract.RequiresForAll(outputs, output => output.IsValid);
Contract.Requires(directoryOutputs.IsValid);
Contract.RequiresForAll(outputs, output => !output.IsSourceFile);
Contract.RequiresForAll(directoryOutputs, directoryOutput => directoryOutput.IsValid);
Contract.Requires(orderDependencies.IsValid);
Contract.RequiresForAll(orderDependencies, dependency => dependency != PipId.Invalid);
Contract.Requires(untrackedPaths.IsValid);
Contract.RequiresForAll(untrackedPaths, path => path.IsValid);
Contract.Requires(untrackedScopes.IsValid);
Contract.RequiresForAll(untrackedScopes, scope => scope.IsValid);
Contract.Requires(!timeout.HasValue || timeout.Value <= MaxTimeout);
Contract.Requires(standardDirectory.IsValid || (standardOutput.IsValid && standardError.IsValid));
Contract.Requires(provenance != null);
Contract.Requires(additionalTempDirectories.IsValid);
Contract.RequiresForAll(additionalTempDirectories, path => path.IsValid);
Contract.Requires(tags.IsValid);
// If the process needs to run in a container, the redirected directory has to be set
Contract.Requires((options & Options.NeedsToRunInContainer) == Options.None || uniqueRedirectedDirectoryRoot.IsValid);
#if DEBUG // a little too expensive for release builds
Contract.Requires(Contract.Exists(dependencies, d => d == executable), "The executable must be declared as a dependency");
Contract.Requires(
!standardInput.IsFile || Contract.Exists(dependencies, d => d == standardInput.File),
"If provided, the standard-input artifact must be declared as a dependency");
Contract.Requires(
!standardOutput.IsValid || Contract.Exists(outputs, o => o.ToFileArtifact() == standardOutput),
"If provided, the standard-error artifact must be declared as an expected output");
Contract.Requires(
!standardError.IsValid || Contract.Exists(outputs, o => o.ToFileArtifact() == standardError),
"If provided, the standard-error artifact must be declared as an expected output");
Contract.Requires(
!responseFile.IsValid ^ responseFileData.IsValid,
"If provided, the response-file artifact must have a corresponding ResponseFileData");
Contract.Requires(outputs.Length == outputs.Distinct().Count());
Contract.Requires(directoryOutputs.Length == directoryOutputs.Distinct().Count());
Contract.Requires(dependencies.Length == dependencies.Distinct().Count());
Contract.Requires(directoryDependencies.Length == directoryDependencies.Distinct().Count());
Contract.Requires(untrackedPaths.Length == untrackedPaths.Distinct().Count());
Contract.Requires(untrackedScopes.Length == untrackedScopes.Distinct().Count());
Contract.Requires(additionalTempDirectories.Length == additionalTempDirectories.Distinct().Count());
Contract.RequiresForAll(semaphores, s => s.IsValid);
Contract.Requires(semaphores.Length == semaphores.Distinct().Count());
#endif
Provenance = provenance;
Tags = tags;
Executable = executable;
ToolDescription = toolDescription;
WorkingDirectory = workingDirectory;
Arguments = arguments;
ResponseFile = responseFile;
ResponseFileData = responseFileData;
StandardOutput = standardOutput;
StandardError = standardError;
StandardInput = standardInput;
StandardDirectory = standardDirectory;
WarningTimeout = warningTimeout;
Timeout = timeout;
// We allow any IEnumerable for these fields, but perform a copy up-front.
// See the remarks of RemoveDuplicateFileArtifacts for why it is used on the input / output lists.
Dependencies = dependencies;
DirectoryDependencies = directoryDependencies;
FileOutputs = outputs;
DirectoryOutputs = directoryOutputs;
OrderDependencies = orderDependencies;
UntrackedPaths = untrackedPaths;
UntrackedScopes = untrackedScopes;
EnvironmentVariables = environmentVariables;
SuccessExitCodes = successExitCodes;
RetryExitCodes = retryExitCodes ?? ReadOnlyArray <int> .Empty;
WarningRegex = warningRegex;
ErrorRegex = errorRegex;
UniqueOutputDirectory = uniqueOutputDirectory;
UniqueRedirectedDirectoryRoot = uniqueRedirectedDirectoryRoot;
Semaphores = semaphores;
TempDirectory = tempDirectory;
TestRetries = testRetries;
ServiceInfo = serviceInfo;
ProcessOptions = options;
AdditionalTempDirectories = additionalTempDirectories;
AllowedSurvivingChildProcessNames = allowedSurvivingChildProcessNames ?? ReadOnlyArray <PathAtom> .Empty;
NestedProcessTerminationTimeout = nestedProcessTerminationTimeout;
ProcessAbsentPathProbeInUndeclaredOpaquesMode = absentPathProbeMode;
DoubleWritePolicy = doubleWritePolicy;
ContainerIsolationLevel = containerIsolationLevel;
}
public void DoubleWriteMakesPipCacheableWhenOutputsAreIsolated(ContainerIsolationLevel containerIsolationLevel, DoubleWritePolicy doubleWritePolicy, bool expectCacheHit, bool expectViolationIsError)
{
string sharedOpaqueDir = Path.Combine(ObjectRoot, "sharedopaquedir");
AbsolutePath sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
FileArtifact doubleWriteArtifact = CreateOutputFileArtifact(sharedOpaqueDir);
ScheduleDoubleWriteProducers(
sharedOpaqueDirPath,
doubleWriteArtifact,
containerIsolationLevel,
doubleWritePolicy,
out ProcessWithOutputs firstProducer,
out ProcessWithOutputs secondProducer);
var firstRunResult = RunScheduler();
if (!expectViolationIsError)
{
firstRunResult.AssertSuccess();
// Run a second time so we can check the caching behavior
var result = RunScheduler().AssertSuccess();
if (expectCacheHit)
{
// In this case, both should be a hit
result.AssertCacheHit(firstProducer.Process.PipId);
result.AssertCacheHit(secondProducer.Process.PipId);
}
else
{
// In this case, the second one should be a miss
result.AssertCacheHit(firstProducer.Process.PipId);
result.AssertCacheMiss(secondProducer.Process.PipId);
AssertWarningEventLogged(LogEventId.ProcessNotStoredToCacheDueToFileMonitoringViolations, 2);
}
// We are expecting a double write as a verbose message (twice, one for each run)
AssertVerboseEventLogged(LogEventId.DependencyViolationDoubleWrite, 2);
}
// The violation is either an error or a warning depending on expectations
if (expectViolationIsError)
{
AssertErrorEventLogged(LogEventId.FileMonitoringError);
AssertErrorEventLogged(ProcessesLogEventId.DisallowedDoubleWriteOnMerge);
}
else
{
AssertWarningEventLogged(LogEventId.FileMonitoringWarning, 2);
}
}
private ProcessBuilder CreateFileInSharedOpaqueBuilder(ContainerIsolationLevel containerIsolationLevel, DoubleWritePolicy doubleWritePolicy, FileArtifact writeArtifact, string writeContent, AbsolutePath sharedOpaqueDirPath)
{
ProcessBuilder producerBuilder;
IEnumerable <Operation> producerWrites =
new Operation[]
{
Operation.CreateDir(new DirectoryArtifact(writeArtifact.Path.GetParent(Context.PathTable), 0, isSharedOpaque: false)),
Operation.WriteFile(writeArtifact, writeContent, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()), // so each builder is unique
};
producerBuilder = CreatePipBuilder(producerWrites);
producerBuilder.AddOutputDirectory(sharedOpaqueDirPath, SealDirectoryKind.SharedOpaque);
producerBuilder.Options |= Process.Options.NeedsToRunInContainer;
producerBuilder.ContainerIsolationLevel = containerIsolationLevel;
producerBuilder.DoubleWritePolicy = doubleWritePolicy;
return(producerBuilder);
}
