/// <summary>
/// Gets the deserialized path set given its content hash
/// </summary>
public virtual async Task <Possible <ObservedPathSet> > TryRetrievePathSetAsync(
OperationContext operationContext,
// TODO: Do we need this fingerprint given that the path set hash is provided by this interface in the first place
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash)
{
using (operationContext.StartOperation(PipExecutorCounter.TryLoadPathSetFromContentCacheDuration))
{
Possible <Stream> maybePathSetStream = await TryLoadAndOpenPathSetStreamAsync(pathSetHash);
if (!maybePathSetStream.Succeeded)
{
return(maybePathSetStream.Failure);
}
using (operationContext.StartOperation(PipExecutorCounter.TryLoadPathSetFromContentCacheDeserializeDuration))
using (var pathSetReader = new BuildXLReader(debug: false, stream: maybePathSetStream.Result, leaveOpen: false))
{
var maybeDeserialized = ObservedPathSet.TryDeserialize(PathTable, pathSetReader, m_pathExpander);
if (!maybeDeserialized.Succeeded)
{
return(maybeDeserialized.Failure);
}
return(maybeDeserialized.Result);
}
}
}
public ProcessStrongFingerprintComputationData(BinaryLogReader.EventReader reader)
{
PathSetHash = reader.ReadContentHash();
var maybePathSet = ObservedPathSet.TryDeserialize(reader.PathTable, reader, pathReader: r => r.ReadAbsolutePath(), stringReader: r => ((BinaryLogReader.EventReader)r).ReadDynamicStringId());
Contract.Assert(maybePathSet.Succeeded);
PathSet = maybePathSet.Result;
PriorStrongFingerprints = reader.ReadReadOnlyList(r => r.ReadStrongFingerprint());
Succeeded = reader.ReadBoolean();
if (Succeeded)
{
IsStrongFingerprintHit = reader.ReadBoolean();
ObservedInputs = reader.ReadReadOnlyArray(r => ObservedInput.Deserialize(r));
ComputedStrongFingerprint = reader.ReadStrongFingerprint();
IsNewlyPublishedAugmentedWeakFingerprint = reader.ReadBoolean();
AugmentedWeakFingerprint = reader.ReadNullableStruct(r => r.ReadWeakFingerprint());
}
else
{
IsStrongFingerprintHit = false;
ObservedInputs = default;
ComputedStrongFingerprint = default;
IsNewlyPublishedAugmentedWeakFingerprint = false;
AugmentedWeakFingerprint = default;
}
}
private static ObservedPathSet SerializeRoundTripAndAssertEquivalent(PathTable pathTable, ObservedPathSet original, PathExpander pathExpander = null)
{
using (var mem = new MemoryStream())
{
using (var writer = new BuildXLWriter(stream: mem, debug: true, leaveOpen: true, logStats: true))
{
original.Serialize(pathTable, writer, pathExpander);
}
mem.Position = 0;
ObservedPathSet roundtrip;
using (var reader = new BuildXLReader(stream: mem, debug: true, leaveOpen: true))
{
var maybeRoundtrip = ObservedPathSet.TryDeserialize(pathTable, reader, pathExpander);
XAssert.IsTrue(maybeRoundtrip.Succeeded, "Failed to deserialize a path set unexpectedly");
roundtrip = maybeRoundtrip.Result;
}
ObservedPathSetTestUtilities.AssertPathSetsEquivalent(original, roundtrip);
return(roundtrip);
}
}
/// <summary>
/// Deserialize result from reader
/// </summary>
public ExecutionResult Deserialize(BuildXLReader reader, uint workerId)
{
int minAbsolutePathValue = reader.ReadInt32();
Contract.Assert(
minAbsolutePathValue == m_maxSerializableAbsolutePathIndex,
"When deserializing for distribution, the minimum absolute path value must match the serialized minimum absolute path value");
var result = (PipResultStatus)reader.ReadByte();
var converged = reader.ReadBoolean();
var numberOfWarnings = reader.ReadInt32Compact();
var weakFingerprint = reader.ReadNullableStruct(r => r.ReadWeakFingerprint());
ProcessPipExecutionPerformance performanceInformation;
if (reader.ReadBoolean())
{
performanceInformation = ProcessPipExecutionPerformance.Deserialize(reader);
// TODO: It looks like this is the wrong class for WorkerId, because the serialized object has always WorkerId == 0.
performanceInformation.WorkerId = workerId;
}
else
{
performanceInformation = null;
}
var outputContent = ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .FromWithoutCopy(ReadOutputContent(reader));
var directoryOutputs = ReadOnlyArray <(DirectoryArtifact, ReadOnlyArray <FileArtifact>)> .FromWithoutCopy(ReadDirectoryOutputs(reader));
var mustBeConsideredPerpetuallyDirty = reader.ReadBoolean();
var dynamicallyObservedFiles = reader.ReadReadOnlyArray(ReadAbsolutePath);
var dynamicallyProbedFiles = reader.ReadReadOnlyArray(ReadAbsolutePath);
var dynamicallyObservedEnumerations = reader.ReadReadOnlyArray(ReadAbsolutePath);
var allowedUndeclaredSourceReads = reader.ReadReadOnlySet(ReadAbsolutePath);
var absentPathProbesUnderOutputDirectories = reader.ReadReadOnlySet(ReadAbsolutePath);
ReportedFileAccess[] fileAccessViolationsNotWhitelisted;
ReportedFileAccess[] whitelistedFileAccessViolations;
ReportedProcess[] reportedProcesses;
ReadReportedProcessesAndFileAccesses(reader, out fileAccessViolationsNotWhitelisted, out whitelistedFileAccessViolations, out reportedProcesses, readPath: m_readPath);
var twoPhaseCachingInfo = ReadTwoPhaseCachingInfo(reader);
var cacheDescriptor = ReadPipCacheDescriptor(reader);
if (cacheDescriptor != null)
{
cacheDescriptor.OutputContentReplicasMiniBloomFilter = reader.ReadUInt32();
}
ObservedPathSet?pathSet = null;
if (reader.ReadBoolean())
{
var maybePathSet = ObservedPathSet.TryDeserialize(m_executionContext.PathTable, reader, pathReader: ReadAbsolutePath);
pathSet = maybePathSet.Succeeded
? (ObservedPathSet?)maybePathSet.Result
: null;
}
CacheLookupPerfInfo cacheLookupCounters = null;
if (reader.ReadBoolean())
{
cacheLookupCounters = CacheLookupPerfInfo.Deserialize(reader);
}
if (!result.IndicatesNoOutput())
{
// Successful result needs to be changed to not materialized because
// the process outputs are not materialized on the local machine.
result = PipResultStatus.NotMaterialized;
}
var pipProperties = ReadPipProperties(reader);
var hasUserRetries = reader.ReadBoolean();
var processExecutionResult = ExecutionResult.CreateSealed(
result,
numberOfWarnings,
outputContent,
directoryOutputs,
performanceInformation,
weakFingerprint,
fileAccessViolationsNotWhitelisted,
whitelistedFileAccessViolations,
mustBeConsideredPerpetuallyDirty,
dynamicallyObservedFiles,
dynamicallyProbedFiles,
dynamicallyObservedEnumerations,
allowedUndeclaredSourceReads,
absentPathProbesUnderOutputDirectories,
twoPhaseCachingInfo,
cacheDescriptor,
converged,
pathSet,
cacheLookupCounters,
pipProperties,
hasUserRetries);
return(processExecutionResult);
}
/// <summary>
/// Takes a strong fingerprint and returns the deserialized contents of
/// the input assertion list file that corresponds to it
/// </summary>
/// <param name="sfp">The strong fingerprint to get the input assertion
/// list file contents for</param>
/// <param name="cacheErrors">Any cache errors that are found will be
/// added to this collection</param>
/// <returns>Deserialized contents of the input assertion list file
/// corresponding to the specified strong fingerprint</returns>
private async Task <string> GetInputAssertionListFileContentsAsync(StrongFingerprint sfp, ConcurrentDictionary <CacheError, int> cacheErrors)
{
// Check for the NoItem
if (sfp.CasElement.Equals(CasHash.NoItem))
{
return(string.Empty);
}
// Pin the input assertion list file
Possible <string, Failure> possibleString = await m_readOnlySession.PinToCasAsync(sfp.CasElement).ConfigureAwait(false);
if (!possibleString.Succeeded)
{
return(string.Empty);
}
// Get the stream for the input assertion list file
Possible <Stream, Failure> possibleStream = await m_readOnlySession.GetStreamAsync(sfp.CasElement).ConfigureAwait(false);
if (!possibleStream.Succeeded)
{
cacheErrors.TryAdd(
new CacheError(
CacheErrorType.CasHashError,
"The input assertion list for SFP " + sfp.ToString() + " was not found in CAS"), 0);
return(string.Empty);
}
// Read the stream contents while hashing
return(await Task.Run(() =>
{
using (var hasher = ContentHashingUtilities.HashInfo.CreateContentHasher())
{
using (var hashingStream = hasher.CreateReadHashingStream(possibleStream.Result))
{
using (var reader = new BuildXLReader(false, hashingStream, false))
{
var maybePathSet = ObservedPathSet.TryDeserialize(s_pathTable, reader);
// Check if deserialization was successful
if (!maybePathSet.Succeeded)
{
// Deserialization failed
cacheErrors.TryAdd(
new CacheError(
CacheErrorType.CasHashError,
"The input assertion list for SFP " + sfp.ToString() + " could not be deserialized"), 0);
return string.Empty;
}
CasHash newCasHash = new CasHash(hashingStream.GetContentHash());
// Check if the hashes match
if (!sfp.CasElement.Equals(newCasHash))
{
cacheErrors.TryAdd(
new CacheError(
CacheErrorType.CasHashError,
"The input assertion list for SFP " + sfp.ToString() + " has been altered in the CAS"), 0);
return string.Empty;
}
// Deserialization was successful and file was unaltered
StringBuilder fileContents = new StringBuilder();
foreach (ObservedPathEntry entry in maybePathSet.Result.Paths)
{
fileContents.Append(entry.Path.ToString(s_pathTable)).Append(Environment.NewLine);
}
return fileContents.ToString();
}
}
}
}).ConfigureAwait(false));
}
/// <summary>
/// Check the input list against the regex
/// </summary>
/// <param name="weak">The weak fingerprint (for logging on failure)</param>
/// <param name="casElement">The CasElement of the strong fingerprint</param>
/// <param name="hashElement">The hashElement of the strong fingerprint (for logging on failure)</param>
/// <param name="urgencyHint">Pass-through</param>
/// <param name="activityId">Pass-through activityId</param>
/// <returns>false if the check was not performed, true if the checks were performed, failure if the regex checks failed</returns>
/// <remarks>
/// This will attempt to validate the CAS stored input list against the regex rules
/// </remarks>
private async Task <Possible <bool, Failure> > CheckInputList(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, UrgencyHint urgencyHint, Guid activityId)
{
// If we either have no CasHash item or we have no regex to check, just return false
// (that we did nothing)
if (casElement.Equals(CasHash.NoItem) || ((Cache.MustIncludeRegex == null) && (Cache.MustNotIncludeRegex == null)))
{
return(false);
}
// mustInclude start out false if we need to check for mustInclude
// Once we get a mustInclude match we no longer need to check.
// If we have no mustInclude regex, we set it to true such that
// we don't bother checking it
bool mustInclude = Cache.MustIncludeRegex == null;
// This is just to make a faster check for the MustNotinclude
// case. If we have the regex then we must check each entry
// but in many cases we don't have the regex so let this be a quick out.
bool checkMustNot = Cache.MustNotIncludeRegex != null;
// Try to get the observed inputs from the CasHash given
var possibleStream = await GetStreamAsync(casElement, urgencyHint, activityId);
if (!possibleStream.Succeeded)
{
// If we could not get a stream to the CasEntery in the fingerprint.
return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, "Failed to get stream of CasElement"));
}
// Deserialize the contents of the path set.
using (possibleStream.Result)
{
PathTable pathTable = new PathTable();
BuildXLReader reader = new BuildXLReader(false, possibleStream.Result, true);
var maybePathSet = ObservedPathSet.TryDeserialize(pathTable, reader);
if (maybePathSet.Succeeded)
{
// Deserialization was successful
foreach (ObservedPathEntry entry in maybePathSet.Result.Paths)
{
string filepath = entry.Path.ToString(pathTable);
// Have we seen a must-have entry yet? If not check if this is one
// that way once we found one we want we stop checking this regex
if (!mustInclude)
{
mustInclude = Cache.MustIncludeRegex.IsMatch(filepath);
}
// Now, if we are looking for a must not include, we just check for that
// and if it matches we fail
if (checkMustNot)
{
if (Cache.MustNotIncludeRegex.IsMatch(filepath))
{
return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, string.Format(CultureInfo.InvariantCulture, "Failed due to a MustNotInclude file: {0}", filepath)));
}
}
}
}
else
{
return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, "Failed to deserialize observed inputs"));
}
}
if (!mustInclude)
{
return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, "Failed due to not including at least one MustInclude file"));
}
return(true);
}
