internal static MutablePipState Deserialize(BuildXLReader reader, PipType pipType, long semiStableHash, PageableStoreId storeId)
{
ServiceInfo serviceInfo = reader.ReadNullable(ServiceInfo.InternalDeserialize);
int options = reader.ReadInt32();
RewritePolicy rewritePolicy = (RewritePolicy)reader.ReadByte();
AbsolutePath executablePath = reader.ReadAbsolutePath();
int priority = reader.ReadInt32();
int preserveOutputTrustLevel = reader.ReadInt32();
return(new ProcessMutablePipState(pipType, semiStableHash, storeId, serviceInfo, (Process.Options)options, rewritePolicy, executablePath, priority, preserveOutputTrustLevel));
}
/// <summary>
/// Sets the strictest defaults for both double write and source rewrite settings if absent, respecting already configured settings.
/// </summary>
public static RewritePolicy StrictDefaultsIfAbsent(this RewritePolicy rewritePolicy)
{
if ((rewritePolicy & RewritePolicy.DoubleWriteMask) == 0)
{
rewritePolicy |= RewritePolicy.DoubleWritesAreErrors;
}
if ((rewritePolicy & RewritePolicy.SourceRewriteMask) == 0)
{
rewritePolicy |= RewritePolicy.SourceRewritesAreErrors;
}
return(rewritePolicy);
}
/// <summary>
/// Checks the policy is valid. There should be at most one double write policy set and one source rewrite policy set
/// </summary>
public static bool IsValid(this RewritePolicy policy)
{
var doubleWrite = policy & RewritePolicy.DoubleWriteMask;
// If the double write policy is not a power of two, then there is more than one set and this is not valid
if (doubleWrite != 0 && (doubleWrite & (doubleWrite - 1)) != 0)
{
return(false);
}
// Same treatment for source rewrite policies
var sourceRewrite = policy & RewritePolicy.SourceRewriteMask;
return(sourceRewrite == 0 || (sourceRewrite & (sourceRewrite - 1)) == 0);
}
/// <summary>
/// Whether the double-write policy implies that double writes are possible without implying a build break
/// </summary>
public static bool ImpliesDoubleWriteAllowed(this RewritePolicy policy)
{
switch (policy & RewritePolicy.DoubleWriteMask)
{
case RewritePolicy.DoubleWritesAreErrors:
return(false);
case RewritePolicy.AllowSameContentDoubleWrites:
case RewritePolicy.UnsafeFirstDoubleWriteWins:
return(true);
default:
throw new InvalidOperationException("Unexpected double write policy " + policy.ToString());
}
}
private void ScheduleDoubleWriteProducers(
AbsolutePath sharedOpaqueDirPath,
FileArtifact doubleWriteArtifact,
ContainerIsolationLevel containerIsolationLevel,
RewritePolicy doubleWritePolicy,
out ProcessWithOutputs firstProducer,
out ProcessWithOutputs secondProducer)
{
var firstProducerBuilder = CreateFileInSharedOpaqueBuilder(containerIsolationLevel, doubleWritePolicy, doubleWriteArtifact, "first", sharedOpaqueDirPath);
firstProducer = SchedulePipBuilder(firstProducerBuilder);
var secondProducerBuilder = CreateFileInSharedOpaqueBuilder(containerIsolationLevel, doubleWritePolicy, doubleWriteArtifact, "second", sharedOpaqueDirPath);
// Let's order this so who is the violator is deterministic
secondProducerBuilder.AddInputDirectory(firstProducer.Process.DirectoryOutputs.First());
secondProducer = SchedulePipBuilder(secondProducerBuilder);
}
internal ProcessMutablePipState(
PipType pipType,
long semiStableHash,
PageableStoreId storeId,
ServiceInfo serviceInfo,
Process.Options processOptions,
RewritePolicy rewritePolicy,
AbsolutePath executablePath,
int priority,
int?preserveOutputsTrustLevel = null)
: base(pipType, semiStableHash, storeId)
{
ServiceInfo = serviceInfo;
ProcessOptions = processOptions;
RewritePolicy = rewritePolicy;
ExecutablePath = executablePath;
Priority = priority;
PreserveOutputTrustLevel = preserveOutputsTrustLevel ?? 0;
}
/// <summary>
/// Adds a fake process pip that produces only the given path.
/// </summary>
public Process AddProcess(AbsolutePath producedPath, RewritePolicy doubleWritePolicy = RewritePolicy.DoubleWritesAreErrors)
{
Contract.Assume(!m_pathProducers.ContainsKey(producedPath), "Each path may have only one producer (no rewrites)");
AbsolutePath workingDirectory = AbsolutePath.Create(m_context.PathTable, PathGeneratorUtilities.GetAbsolutePath("X", ""));
AbsolutePath exe = AbsolutePath.Create(m_context.PathTable, PathGeneratorUtilities.GetAbsolutePath("X", "fake.exe"));
var process = new Process(
executable: FileArtifact.CreateSourceFile(exe),
workingDirectory: workingDirectory,
arguments: PipDataBuilder.CreatePipData(m_context.StringTable, string.Empty, PipDataFragmentEscaping.NoEscaping),
responseFile: FileArtifact.Invalid,
responseFileData: PipData.Invalid,
environmentVariables: ReadOnlyArray <EnvironmentVariable> .Empty,
standardInput: FileArtifact.Invalid,
standardOutput: FileArtifact.Invalid,
standardError: FileArtifact.Invalid,
standardDirectory: workingDirectory,
warningTimeout: null,
timeout: null,
dependencies: ReadOnlyArray <FileArtifact> .FromWithoutCopy(FileArtifact.CreateSourceFile(exe)),
outputs: ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(FileArtifact.CreateSourceFile(producedPath).CreateNextWrittenVersion().WithAttributes()),
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty,
directoryOutputs: ReadOnlyArray <DirectoryArtifact> .Empty,
orderDependencies: ReadOnlyArray <PipId> .Empty,
untrackedPaths: ReadOnlyArray <AbsolutePath> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .Empty,
tags: ReadOnlyArray <StringId> .Empty,
successExitCodes: ReadOnlyArray <int> .Empty,
semaphores: ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(m_context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty,
rewritePolicy: doubleWritePolicy);
process.PipId = AllocateNextPipId();
m_pips.Add(process.PipId, process);
m_pathProducers.Add(producedPath, process);
return(process);
}
internal ProcessMutablePipState(
PipType pipType,
long semiStableHash,
PageableStoreId storeId,
ServiceInfo serviceInfo,
Process.Options processOptions,
RewritePolicy rewritePolicy,
AbsolutePath executablePath,
int priority,
ModuleId moduleId,
int preserveOutputsTrustLevel = 0,
bool isSucceedFast = false)
: base(pipType, semiStableHash, storeId)
{
ServiceInfo = serviceInfo;
ProcessOptions = processOptions;
RewritePolicy = rewritePolicy;
ExecutablePath = executablePath;
Priority = priority;
PreserveOutputTrustLevel = preserveOutputsTrustLevel;
ModuleId = moduleId;
IsSucceedFast = isSucceedFast;
}
public void DoubleWriteMakesPipCacheableWhenOutputsAreIsolated(ContainerIsolationLevel containerIsolationLevel, RewritePolicy doubleWritePolicy, bool expectCacheHit, bool expectViolationIsError)
{
string sharedOpaqueDir = Path.Combine(ObjectRoot, "sharedopaquedir");
AbsolutePath sharedOpaqueDirPath = AbsolutePath.Create(Context.PathTable, sharedOpaqueDir);
FileArtifact doubleWriteArtifact = CreateOutputFileArtifact(sharedOpaqueDir);
ScheduleDoubleWriteProducers(
sharedOpaqueDirPath,
doubleWriteArtifact,
containerIsolationLevel,
doubleWritePolicy,
out ProcessWithOutputs firstProducer,
out ProcessWithOutputs secondProducer);
var firstRunResult = RunScheduler();
if (!expectViolationIsError)
{
firstRunResult.AssertSuccess();
// Run a second time so we can check the caching behavior
var result = RunScheduler().AssertSuccess();
if (expectCacheHit)
{
// In this case, both should be a hit
result.AssertCacheHit(firstProducer.Process.PipId);
result.AssertCacheHit(secondProducer.Process.PipId);
}
else
{
// In this case, the second one should be a miss
result.AssertCacheHit(firstProducer.Process.PipId);
result.AssertCacheMiss(secondProducer.Process.PipId);
AssertWarningEventLogged(LogEventId.ProcessNotStoredToCacheDueToFileMonitoringViolations, 2);
}
// We are expecting a double write as a verbose message (twice, one for each run)
AssertVerboseEventLogged(LogEventId.DependencyViolationDoubleWrite, 2);
}
// The violation is either an error or a warning depending on expectations
if (expectViolationIsError)
{
AssertErrorEventLogged(LogEventId.FileMonitoringError);
AssertErrorEventLogged(ProcessesLogEventId.DisallowedDoubleWriteOnMerge);
}
else
{
AssertWarningEventLogged(LogEventId.FileMonitoringWarning, 2);
}
}
private ProcessBuilder CreateFileInSharedOpaqueBuilder(ContainerIsolationLevel containerIsolationLevel, RewritePolicy rewritePolicy, FileArtifact writeArtifact, string writeContent, AbsolutePath sharedOpaqueDirPath)
{
ProcessBuilder producerBuilder;
IEnumerable <Operation> producerWrites =
new Operation[]
{
Operation.CreateDir(new DirectoryArtifact(writeArtifact.Path.GetParent(Context.PathTable), 0, isSharedOpaque: false)),
Operation.WriteFile(writeArtifact, writeContent, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()), // so each builder is unique
};
producerBuilder = CreatePipBuilder(producerWrites);
producerBuilder.AddOutputDirectory(sharedOpaqueDirPath, SealDirectoryKind.SharedOpaque);
producerBuilder.Options |= Process.Options.NeedsToRunInContainer;
producerBuilder.ContainerIsolationLevel = containerIsolationLevel;
producerBuilder.RewritePolicy = rewritePolicy;
return(producerBuilder);
}
/// <summary>
/// Whether the policy implies that produced content defines what is allowed/denied
/// </summary>
public static bool ImpliesContentAwareness(this RewritePolicy policy)
{
return((policy & RewritePolicy.AllowSameContentDoubleWrites) != 0 || (policy & RewritePolicy.SafeSourceRewritesAreAllowed) != 0);
}
