private async Task <CasHash> AddPathSet(ICacheSession session, params string[] thePaths)
{
var pathTable = new PathTable();
ObservedPathEntry[] paths = new ObservedPathEntry[thePaths.Length];
for (int i = 0; i < thePaths.Length; i++)
{
AbsolutePath absPath = AbsolutePath.Create(pathTable, thePaths[i]);
paths[i] = new ObservedPathEntry(absPath, false, false, false, null, false);
}
var emptyObservedAccessFileNames = SortedReadOnlyArray <StringId, CaseInsensitiveStringIdComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <StringId> .Empty,
new CaseInsensitiveStringIdComparer(pathTable.StringTable));
ObservedPathSet pathSet = new ObservedPathSet(
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <ObservedPathEntry> .FromWithoutCopy(paths),
new ObservedPathEntryExpandedPathComparer(pathTable.ExpandedPathComparer)),
emptyObservedAccessFileNames,
null);
using (var pathSetBuffer = new MemoryStream())
{
using (var writer = new BuildXLWriter(stream: pathSetBuffer, debug: false, leaveOpen: true, logStats: false))
{
pathSet.Serialize(pathTable, writer, preserveCasing: false);
}
pathSetBuffer.Seek(0, SeekOrigin.Begin);
// Must await such that the dispose of the MemoryStream is only after the write completes
return(await session.AddToCasAsync(pathSetBuffer).SuccessAsync());
}
}
public ProcessStrongFingerprintComputationData(BinaryLogReader.EventReader reader)
{
PathSetHash = reader.ReadContentHash();
var maybePathSet = ObservedPathSet.TryDeserialize(reader.PathTable, reader, pathReader: r => r.ReadAbsolutePath(), stringReader: r => ((BinaryLogReader.EventReader)r).ReadDynamicStringId());
Contract.Assert(maybePathSet.Succeeded);
PathSet = maybePathSet.Result;
PriorStrongFingerprints = reader.ReadReadOnlyList(r => r.ReadStrongFingerprint());
Succeeded = reader.ReadBoolean();
if (Succeeded)
{
IsStrongFingerprintHit = reader.ReadBoolean();
ObservedInputs = reader.ReadReadOnlyArray(r => ObservedInput.Deserialize(r));
ComputedStrongFingerprint = reader.ReadStrongFingerprint();
IsNewlyPublishedAugmentedWeakFingerprint = reader.ReadBoolean();
AugmentedWeakFingerprint = reader.ReadNullableStruct(r => r.ReadWeakFingerprint());
}
else
{
IsStrongFingerprintHit = false;
ObservedInputs = default;
ComputedStrongFingerprint = default;
IsNewlyPublishedAugmentedWeakFingerprint = false;
AugmentedWeakFingerprint = default;
}
}
public static void AssertPathSetContainsDuplicates(ObservedPathSet pathSet)
{
bool foundDuplicate = false;
for (int i = 1; i < pathSet.Paths.Length; i++)
{
int cmp = pathSet.Paths.Comparer.Compare(pathSet.Paths[i], pathSet.Paths[i - 1]);
XAssert.IsTrue(
cmp >= 0,
"Path set must contain unique items in a sorted order");
foundDuplicate |= cmp == 0;
}
for (int i = 1; i < pathSet.ObservedAccessedFileNames.Length; i++)
{
int cmp = pathSet.ObservedAccessedFileNames.Comparer.Compare(pathSet.ObservedAccessedFileNames[i], pathSet.ObservedAccessedFileNames[i - 1]);
XAssert.IsTrue(
cmp >= 0,
"Observed accessed file names must contain unique items in a sorted order");
foundDuplicate |= cmp == 0;
}
XAssert.IsTrue(foundDuplicate, "Expected at least one duplicate");
}
/// <summary>
/// Stores path set for pip for use during two phase lookup
/// </summary>
public virtual Task <Possible <ContentHash> > TryStorePathSetAsync(ObservedPathSet pathSet)
{
return(this.TrySerializedAndStorePathSetAsync(pathSet, (pathSetHash, pathSetBuffer) =>
{
return TryStorePathSetContentAsync(pathSetHash, pathSetBuffer);
}));
}
/// <summary>
/// Stores path set for pip for use during two phase lookup
/// </summary>
public virtual Task <Possible <ContentHash> > TryStorePathSetAsync(ObservedPathSet pathSet, bool preservePathCasing)
{
return(TrySerializedAndStorePathSetAsync(pathSet, (pathSetHash, pathSetBuffer) =>
{
return TryStorePathSetContentAsync(pathSetHash, pathSetBuffer);
}, preservePathCasing));
}
/// <summary>
/// Serializes a path set.
/// </summary>
public async Task <ContentHash> SerializePathSetAsync(ObservedPathSet pathSet, bool preservePathCasing)
{
using (var pathSetBuffer = new MemoryStream())
{
return(await SerializePathSetAsync(pathSet, pathSetBuffer, preservePathCasing));
}
}
/// <summary>
/// Serializes a path set.
/// </summary>
public async Task <ContentHash> SerializePathSet(ObservedPathSet pathSet)
{
using (var pathSetBuffer = new MemoryStream())
{
return(await SerializePathSet(pathSet, pathSetBuffer));
}
}
/// <summary>
/// Gets the deserialized path set given its content hash
/// </summary>
public virtual async Task <Possible <ObservedPathSet> > TryRetrievePathSetAsync(
OperationContext operationContext,
// TODO: Do we need this fingerprint given that the path set hash is provided by this interface in the first place
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash)
{
using (operationContext.StartOperation(PipExecutorCounter.TryLoadPathSetFromContentCacheDuration))
{
Possible <Stream> maybePathSetStream = await TryLoadAndOpenPathSetStreamAsync(pathSetHash);
if (!maybePathSetStream.Succeeded)
{
return(maybePathSetStream.Failure);
}
using (operationContext.StartOperation(PipExecutorCounter.TryLoadPathSetFromContentCacheDeserializeDuration))
using (var pathSetReader = new BuildXLReader(debug: false, stream: maybePathSetStream.Result, leaveOpen: false))
{
var maybeDeserialized = ObservedPathSet.TryDeserialize(PathTable, pathSetReader, m_pathExpander);
if (!maybeDeserialized.Succeeded)
{
return(maybeDeserialized.Failure);
}
return(maybeDeserialized.Result);
}
}
}
public static void AssertPathSetDoesNotContainDuplicates(ObservedPathSet pathSet)
{
for (int i = 1; i < pathSet.Paths.Length; i++)
{
XAssert.IsTrue(
pathSet.Paths.Comparer.Compare(pathSet.Paths[i], pathSet.Paths[i - 1]) > 0,
"Path set must contain unique items in a sorted order");
}
}
private static void AssertPathSetEquals(PathTable pathTable1, ObservedPathSet pathSet1, PathTable pathTable2, ObservedPathSet pathSet2)
{
Assert.Equal(pathSet1.Paths.Length, pathSet2.Paths.Length);
for (int i = 0; i < pathSet1.Paths.Length; i++)
{
Assert.Equal(
pathSet1.Paths[i].Path.ToString(pathTable1).ToUpperInvariant(),
pathSet2.Paths[i].Path.ToString(pathTable2).ToUpperInvariant());
}
}
public static void AssertPathSetsEquivalent(ObservedPathSet a, ObservedPathSet b)
{
List <AbsolutePath> aList = RemoveDuplicates(a);
List <AbsolutePath> bList = RemoveDuplicates(b);
XAssert.AreEqual(aList.Count, bList.Count);
for (int i = 0; i < aList.Count; i++)
{
XAssert.AreEqual(aList[i], bList[i]);
}
}
/// <summary>
/// Clones the current strong fingerprint computation replacing the path set and observed inputs.
/// Internal use only for converting data to refer to different graph data structures.
/// </summary>
internal ProcessStrongFingerprintComputationData UnsafeOverride(
ObservedPathSet pathSet,
ReadOnlyArray <ObservedInput> observedInputs)
{
var data = this;
// a new PathSet might have a hash than is different from the one recorded in PathSetHash
data.PathSet = pathSet;
data.ObservedInputs = observedInputs;
return(data);
}
/// <nodoc />
public static Xldb.ObservedPathSet ToObservedPathSet(this ObservedPathSet pathSet, PathTable pathTable)
{
var observedPathSet = new Xldb.ObservedPathSet();
observedPathSet.Paths.AddRange(pathSet.Paths.Select(pathEntry => pathEntry.ToObservedPathEntry(pathTable)));
observedPathSet.ObservedAccessedFileNames.AddRange(
pathSet.ObservedAccessedFileNames.Select(
observedAccessedFileName => new Xldb.StringId()
{
Value = observedAccessedFileName.Value
}));
observedPathSet.UnsafeOptions = pathSet.UnsafeOptions.ToUnsafeOptions();
return(observedPathSet);
}
/// <nodoc />
public ProcessStrongFingerprintComputationData(
ContentHash pathSetHash,
List <StrongContentFingerprint> priorStrongFingerprints,
ObservedPathSet pathSet)
{
PathSetHash = pathSetHash;
PathSet = pathSet;
PriorStrongFingerprints = priorStrongFingerprints;
// Initial defaults
Succeeded = false;
IsStrongFingerprintHit = false;
ObservedInputs = default(ReadOnlyArray <ObservedInput>);
ComputedStrongFingerprint = default(StrongContentFingerprint);
}
public static List <AbsolutePath> RemoveDuplicates(ObservedPathSet pathSet)
{
var paths = new List <AbsolutePath>();
for (int i = 0; i < pathSet.Paths.Length; i++)
{
while (i + 1 < pathSet.Paths.Length && pathSet.Paths[i + 1].Path == pathSet.Paths[i].Path)
{
i++;
}
paths.Add(pathSet.Paths[i].Path);
}
return(paths);
}
/// <nodoc />
public ProcessStrongFingerprintComputationData(
ContentHash pathSetHash,
List <StrongContentFingerprint> priorStrongFingerprints,
ObservedPathSet pathSet)
{
PathSetHash = pathSetHash;
PathSet = pathSet;
PriorStrongFingerprints = priorStrongFingerprints;
// Initial defaults
Succeeded = false;
IsStrongFingerprintHit = false;
ObservedInputs = default;
ComputedStrongFingerprint = default;
IsNewlyPublishedAugmentedWeakFingerprint = false;
AugmentedWeakFingerprint = default;
}
/// <summary>
/// Serializes a path set to the given buffer.
/// </summary>
protected async Task <ContentHash> SerializePathSet(ObservedPathSet pathSet, MemoryStream pathSetBuffer, ContentHash?pathSetHash = null)
{
using (var writer = new BuildXLWriter(stream: pathSetBuffer, debug: false, leaveOpen: true, logStats: false))
{
pathSet.Serialize(PathTable, writer, m_pathExpander);
if (pathSetHash == null)
{
pathSetBuffer.Position = 0;
pathSetHash = await ContentHashingUtilities.HashContentStreamAsync(pathSetBuffer);
}
pathSetBuffer.Position = 0;
return(pathSetHash.Value);
}
}
private ProcessStrongFingerprintComputationData Convert(ProcessStrongFingerprintComputationData computation)
{
if (AreGraphsSame)
{
return(computation);
}
var pathSet = new ObservedPathSet(
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> .FromSortedArrayUnsafe(
Convert(computation.PathEntries, this, (i, me) => me.Convert(i)),
new ObservedPathEntryExpandedPathComparer(OldModel.PathTable.ExpandedPathComparer)),
computation.PathSet.ObservedAccessedFileNames,
computation.PathSet.UnsafeOptions);
return(computation.UnsafeOverride(
pathSet,
Convert(computation.ObservedInputs, this, (i, me) => me.Convert(i))));
}
public void ProjectPathSet()
{
var pathTable = new PathTable();
AbsolutePath firstPath = AbsolutePath.Create(pathTable, X("/X/bar"));
AbsolutePath secondPath = AbsolutePath.Create(pathTable, X("/X/foo"));
var p = CreateResult(
pathTable,
ObservedInput.CreateFileContentRead(firstPath, ContentHashingUtilities.EmptyHash),
ObservedInput.CreateFileContentRead(secondPath, ContentHashingUtilities.EmptyHash));
ObservedPathSet projected = p.GetPathSet(unsafeOptions: null);
ObservedPathSetTestUtilities.AssertPathSetsEquivalent(
projected,
ObservedPathSetTestUtilities.CreatePathSet(pathTable, firstPath, secondPath));
}
public async Task TokenizedPathSet()
{
var pathTable = new PathTable();
var pathExpanderA = new MountPathExpander(pathTable);
AddMount(pathExpanderA, pathTable, AbsolutePath.Create(pathTable, X("/x/users/AUser")), "UserProfile", isSystem: true);
AddMount(pathExpanderA, pathTable, AbsolutePath.Create(pathTable, X("/x/windows")), "Windows", isSystem: true);
AddMount(pathExpanderA, pathTable, AbsolutePath.Create(pathTable, X("/x/test")), "TestRoot", isSystem: false);
var pathSetA = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/x/abc"),
X("/x/users/AUser/def"),
X("/x/windows"),
X("/x/test/abc"));
ObservedPathSet roundtripA = SerializeRoundTripAndAssertEquivalent(pathTable, pathSetA);
XAssert.AreEqual(4, roundtripA.Paths.Length);
ContentHash pathSetHashA = await pathSetA.ToContentHash(pathTable, pathExpanderA);
var pathExpanderB = new MountPathExpander(pathTable);
AddMount(pathExpanderB, pathTable, AbsolutePath.Create(pathTable, X("/y/users/BUser")), "UserProfile", isSystem: true);
AddMount(pathExpanderB, pathTable, AbsolutePath.Create(pathTable, X("/y/windows")), "Windows", isSystem: true);
AddMount(pathExpanderB, pathTable, AbsolutePath.Create(pathTable, X("/y/abc/test")), "TestRoot", isSystem: false);
var pathSetB = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/x/abc"),
X("/y/users/BUser/def"),
X("/y/windows"),
X("/y/abc/test/abc"));
ObservedPathSet roundtripB = SerializeRoundTripAndAssertEquivalent(pathTable, pathSetB);
XAssert.AreEqual(4, roundtripB.Paths.Length);
ContentHash pathSetHashB = await pathSetB.ToContentHash(pathTable, pathExpanderB);
AssertTrue(pathSetHashA == pathSetHashB);
}
private static void AssertCompressedSizeExpected(PathTable pathTable, ObservedPathSet pathSet, params string[] uncompressedStrings)
{
long compressedSize = GetSizeOfSerializedContent(writer => pathSet.Serialize(pathTable, writer));
int numberOfUniquePaths = ObservedPathSetTestUtilities.RemoveDuplicates(pathSet).Count;
// This is correct assuming the following:
// - Each string can be represented with a one byte length prefix, and a one byte reuse-count.
// - The number of strings can be represented in one byte.
// - Each character takes one byte when UTF8 encoded.
long expectedCompressedSize =
GetSizeOfSerializedContent(writer => pathSet.UnsafeOptions.Serialize(writer)) +
1 + // The number of observed accesses file names (0)
1 + // String count
(3 * numberOfUniquePaths) + // Length isSearchPath, isDirectoryPath, and reuse
uncompressedStrings.Sum(s => s.Length);
XAssert.AreEqual(expectedCompressedSize, compressedSize, "Wrong size for compressed path-set");
}
/// <summary>
/// Stores content for the given path set using the given store function
/// </summary>
protected async Task <Possible <ContentHash> > TrySerializedAndStorePathSetAsync(
ObservedPathSet pathSet,
Func <ContentHash, MemoryStream, Task <Possible <Unit> > > storeAsync,
ContentHash?pathSetHash = null)
{
using (var pathSetBuffer = new MemoryStream())
{
var hash = await SerializePathSet(pathSet, pathSetBuffer, pathSetHash);
var maybeStored = await storeAsync(hash, pathSetBuffer);
if (!maybeStored.Succeeded)
{
return(maybeStored.Failure);
}
return(hash);
}
}
/// <summary>
/// Creates a strong fingerprint computation for process execution
/// </summary>
public static ProcessStrongFingerprintComputationData CreateForExecution(
ContentHash pathSetHash,
ObservedPathSet pathSet,
ReadOnlyArray <ObservedInput> observedInputs,
StrongContentFingerprint strongFingerprint)
{
var data = default(ProcessStrongFingerprintComputationData);
data.PathSetHash = pathSetHash;
data.PathSet = pathSet;
data.PriorStrongFingerprints = CollectionUtilities.EmptyArray <StrongContentFingerprint>();
// Initial defaults
data.Succeeded = true;
data.IsStrongFingerprintHit = false;
data.ObservedInputs = observedInputs;
data.ComputedStrongFingerprint = strongFingerprint;
return(data);
}
public static void AssertPathSetsEquivalent(ObservedPathSet a, ObservedPathSet b)
{
List <AbsolutePath> aPaths = RemoveDuplicates(a.Paths);
List <AbsolutePath> bPaths = RemoveDuplicates(b.Paths);
XAssert.AreEqual(aPaths.Count, bPaths.Count);
for (int i = 0; i < aPaths.Count; i++)
{
XAssert.AreEqual(aPaths[i], bPaths[i]);
}
List <StringId> aFileNames = RemoveDuplicates(a.ObservedAccessedFileNames);
List <StringId> bFileNames = RemoveDuplicates(b.ObservedAccessedFileNames);
XAssert.AreEqual(aFileNames.Count, bFileNames.Count);
for (int i = 0; i < aFileNames.Count; i++)
{
XAssert.IsTrue(a.ObservedAccessedFileNames.Comparer.Compare(aFileNames[i], bFileNames[i]) == 0);
}
}
private static ObservedPathSet SerializeRoundTripAndAssertEquivalent(PathTable pathTable, ObservedPathSet original, PathExpander pathExpander = null)
{
using (var mem = new MemoryStream())
{
using (var writer = new BuildXLWriter(stream: mem, debug: true, leaveOpen: true, logStats: true))
{
original.Serialize(pathTable, writer, pathExpander);
}
mem.Position = 0;
ObservedPathSet roundtrip;
using (var reader = new BuildXLReader(stream: mem, debug: true, leaveOpen: true))
{
var maybeRoundtrip = ObservedPathSet.TryDeserialize(pathTable, reader, pathExpander);
XAssert.IsTrue(maybeRoundtrip.Succeeded, "Failed to deserialize a path set unexpectedly");
roundtrip = maybeRoundtrip.Result;
}
ObservedPathSetTestUtilities.AssertPathSetsEquivalent(original, roundtrip);
return(roundtrip);
}
}
/// <summary>
/// Check the input list against the regex
/// </summary>
/// <param name="weak">The weak fingerprint (for logging on failure)</param>
/// <param name="casElement">The CasElement of the strong fingerprint</param>
/// <param name="hashElement">The hashElement of the strong fingerprint (for logging on failure)</param>
/// <param name="urgencyHint">Pass-through</param>
/// <param name="activityId">Pass-through activityId</param>
/// <returns>false if the check was not performed, true if the checks were performed, failure if the regex checks failed</returns>
/// <remarks>
/// This will attempt to validate the CAS stored input list against the regex rules
/// </remarks>
private async Task <Possible <bool, Failure> > CheckInputList(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, UrgencyHint urgencyHint, Guid activityId)
{
// If we either have no CasHash item or we have no regex to check, just return false
// (that we did nothing)
if (casElement.Equals(CasHash.NoItem) || ((Cache.MustIncludeRegex == null) && (Cache.MustNotIncludeRegex == null)))
{
return(false);
}
// mustInclude start out false if we need to check for mustInclude
// Once we get a mustInclude match we no longer need to check.
// If we have no mustInclude regex, we set it to true such that
// we don't bother checking it
bool mustInclude = Cache.MustIncludeRegex == null;
// This is just to make a faster check for the MustNotinclude
// case. If we have the regex then we must check each entry
// but in many cases we don't have the regex so let this be a quick out.
bool checkMustNot = Cache.MustNotIncludeRegex != null;
// Try to get the observed inputs from the CasHash given
var possibleStream = await GetStreamAsync(casElement, urgencyHint, activityId);
if (!possibleStream.Succeeded)
{
// If we could not get a stream to the CasEntery in the fingerprint.
return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, "Failed to get stream of CasElement"));
}
// Deserialize the contents of the path set.
using (possibleStream.Result)
{
PathTable pathTable = new PathTable();
BuildXLReader reader = new BuildXLReader(false, possibleStream.Result, true);
var maybePathSet = ObservedPathSet.TryDeserialize(pathTable, reader);
if (maybePathSet.Succeeded)
{
// Deserialization was successful
foreach (ObservedPathEntry entry in maybePathSet.Result.Paths)
{
string filepath = entry.Path.ToString(pathTable);
// Have we seen a must-have entry yet? If not check if this is one
// that way once we found one we want we stop checking this regex
if (!mustInclude)
{
mustInclude = Cache.MustIncludeRegex.IsMatch(filepath);
}
// Now, if we are looking for a must not include, we just check for that
// and if it matches we fail
if (checkMustNot)
{
if (Cache.MustNotIncludeRegex.IsMatch(filepath))
{
return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, string.Format(CultureInfo.InvariantCulture, "Failed due to a MustNotInclude file: {0}", filepath)));
}
}
}
}
else
{
return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, "Failed to deserialize observed inputs"));
}
}
if (!mustInclude)
{
return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, "Failed due to not including at least one MustInclude file"));
}
return(true);
}
/// <summary>
/// Takes a strong fingerprint and returns the deserialized contents of
/// the input assertion list file that corresponds to it
/// </summary>
/// <param name="sfp">The strong fingerprint to get the input assertion
/// list file contents for</param>
/// <param name="cacheErrors">Any cache errors that are found will be
/// added to this collection</param>
/// <returns>Deserialized contents of the input assertion list file
/// corresponding to the specified strong fingerprint</returns>
private async Task <string> GetInputAssertionListFileContentsAsync(StrongFingerprint sfp, ConcurrentDictionary <CacheError, int> cacheErrors)
{
// Check for the NoItem
if (sfp.CasElement.Equals(CasHash.NoItem))
{
return(string.Empty);
}
// Pin the input assertion list file
Possible <string, Failure> possibleString = await m_readOnlySession.PinToCasAsync(sfp.CasElement).ConfigureAwait(false);
if (!possibleString.Succeeded)
{
return(string.Empty);
}
// Get the stream for the input assertion list file
Possible <Stream, Failure> possibleStream = await m_readOnlySession.GetStreamAsync(sfp.CasElement).ConfigureAwait(false);
if (!possibleStream.Succeeded)
{
cacheErrors.TryAdd(
new CacheError(
CacheErrorType.CasHashError,
"The input assertion list for SFP " + sfp.ToString() + " was not found in CAS"), 0);
return(string.Empty);
}
// Read the stream contents while hashing
return(await Task.Run(() =>
{
using (var hasher = ContentHashingUtilities.HashInfo.CreateContentHasher())
{
using (var hashingStream = hasher.CreateReadHashingStream(possibleStream.Result))
{
using (var reader = new BuildXLReader(false, hashingStream, false))
{
var maybePathSet = ObservedPathSet.TryDeserialize(s_pathTable, reader);
// Check if deserialization was successful
if (!maybePathSet.Succeeded)
{
// Deserialization failed
cacheErrors.TryAdd(
new CacheError(
CacheErrorType.CasHashError,
"The input assertion list for SFP " + sfp.ToString() + " could not be deserialized"), 0);
return string.Empty;
}
CasHash newCasHash = new CasHash(hashingStream.GetContentHash());
// Check if the hashes match
if (!sfp.CasElement.Equals(newCasHash))
{
cacheErrors.TryAdd(
new CacheError(
CacheErrorType.CasHashError,
"The input assertion list for SFP " + sfp.ToString() + " has been altered in the CAS"), 0);
return string.Empty;
}
// Deserialization was successful and file was unaltered
StringBuilder fileContents = new StringBuilder();
foreach (ObservedPathEntry entry in maybePathSet.Result.Paths)
{
fileContents.Append(entry.Path.ToString(s_pathTable)).Append(Environment.NewLine);
}
return fileContents.ToString();
}
}
}
}).ConfigureAwait(false));
}
/// <summary>
/// Deserialize result from reader
/// </summary>
public ExecutionResult Deserialize(BuildXLReader reader, uint workerId)
{
int minAbsolutePathValue = reader.ReadInt32();
Contract.Assert(
minAbsolutePathValue == m_maxSerializableAbsolutePathIndex,
"When deserializing for distribution, the minimum absolute path value must match the serialized minimum absolute path value");
var result = (PipResultStatus)reader.ReadByte();
var converged = reader.ReadBoolean();
var numberOfWarnings = reader.ReadInt32Compact();
var weakFingerprint = reader.ReadNullableStruct(r => r.ReadWeakFingerprint());
ProcessPipExecutionPerformance performanceInformation;
if (reader.ReadBoolean())
{
performanceInformation = ProcessPipExecutionPerformance.Deserialize(reader);
// TODO: It looks like this is the wrong class for WorkerId, because the serialized object has always WorkerId == 0.
performanceInformation.WorkerId = workerId;
}
else
{
performanceInformation = null;
}
var outputContent = ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .FromWithoutCopy(ReadOutputContent(reader));
var directoryOutputs = ReadOnlyArray <(DirectoryArtifact, ReadOnlyArray <FileArtifact>)> .FromWithoutCopy(ReadDirectoryOutputs(reader));
var mustBeConsideredPerpetuallyDirty = reader.ReadBoolean();
var dynamicallyObservedFiles = reader.ReadReadOnlyArray(ReadAbsolutePath);
var dynamicallyProbedFiles = reader.ReadReadOnlyArray(ReadAbsolutePath);
var dynamicallyObservedEnumerations = reader.ReadReadOnlyArray(ReadAbsolutePath);
var allowedUndeclaredSourceReads = reader.ReadReadOnlySet(ReadAbsolutePath);
var absentPathProbesUnderOutputDirectories = reader.ReadReadOnlySet(ReadAbsolutePath);
ReportedFileAccess[] fileAccessViolationsNotWhitelisted;
ReportedFileAccess[] whitelistedFileAccessViolations;
ReportedProcess[] reportedProcesses;
ReadReportedProcessesAndFileAccesses(reader, out fileAccessViolationsNotWhitelisted, out whitelistedFileAccessViolations, out reportedProcesses, readPath: m_readPath);
var twoPhaseCachingInfo = ReadTwoPhaseCachingInfo(reader);
var cacheDescriptor = ReadPipCacheDescriptor(reader);
if (cacheDescriptor != null)
{
cacheDescriptor.OutputContentReplicasMiniBloomFilter = reader.ReadUInt32();
}
ObservedPathSet?pathSet = null;
if (reader.ReadBoolean())
{
var maybePathSet = ObservedPathSet.TryDeserialize(m_executionContext.PathTable, reader, pathReader: ReadAbsolutePath);
pathSet = maybePathSet.Succeeded
? (ObservedPathSet?)maybePathSet.Result
: null;
}
CacheLookupPerfInfo cacheLookupCounters = null;
if (reader.ReadBoolean())
{
cacheLookupCounters = CacheLookupPerfInfo.Deserialize(reader);
}
if (!result.IndicatesNoOutput())
{
// Successful result needs to be changed to not materialized because
// the process outputs are not materialized on the local machine.
result = PipResultStatus.NotMaterialized;
}
var pipProperties = ReadPipProperties(reader);
var hasUserRetries = reader.ReadBoolean();
var processExecutionResult = ExecutionResult.CreateSealed(
result,
numberOfWarnings,
outputContent,
directoryOutputs,
performanceInformation,
weakFingerprint,
fileAccessViolationsNotWhitelisted,
whitelistedFileAccessViolations,
mustBeConsideredPerpetuallyDirty,
dynamicallyObservedFiles,
dynamicallyProbedFiles,
dynamicallyObservedEnumerations,
allowedUndeclaredSourceReads,
absentPathProbesUnderOutputDirectories,
twoPhaseCachingInfo,
cacheDescriptor,
converged,
pathSet,
cacheLookupCounters,
pipProperties,
hasUserRetries);
return(processExecutionResult);
}