/// <summary> /// Gets the deserialized path set given its content hash /// </summary> public virtual async Task <Possible <ObservedPathSet> > TryRetrievePathSetAsync( OperationContext operationContext, // TODO: Do we need this fingerprint given that the path set hash is provided by this interface in the first place WeakContentFingerprint weakFingerprint, ContentHash pathSetHash) { using (operationContext.StartOperation(PipExecutorCounter.TryLoadPathSetFromContentCacheDuration)) { Possible <Stream> maybePathSetStream = await TryLoadAndOpenPathSetStreamAsync(pathSetHash); if (!maybePathSetStream.Succeeded) { return(maybePathSetStream.Failure); } using (operationContext.StartOperation(PipExecutorCounter.TryLoadPathSetFromContentCacheDeserializeDuration)) using (var pathSetReader = new BuildXLReader(debug: false, stream: maybePathSetStream.Result, leaveOpen: false)) { var maybeDeserialized = ObservedPathSet.TryDeserialize(PathTable, pathSetReader, m_pathExpander); if (!maybeDeserialized.Succeeded) { return(maybeDeserialized.Failure); } return(maybeDeserialized.Result); } } }
public ProcessStrongFingerprintComputationData(BinaryLogReader.EventReader reader) { PathSetHash = reader.ReadContentHash(); var maybePathSet = ObservedPathSet.TryDeserialize(reader.PathTable, reader, pathReader: r => r.ReadAbsolutePath(), stringReader: r => ((BinaryLogReader.EventReader)r).ReadDynamicStringId()); Contract.Assert(maybePathSet.Succeeded); PathSet = maybePathSet.Result; PriorStrongFingerprints = reader.ReadReadOnlyList(r => r.ReadStrongFingerprint()); Succeeded = reader.ReadBoolean(); if (Succeeded) { IsStrongFingerprintHit = reader.ReadBoolean(); ObservedInputs = reader.ReadReadOnlyArray(r => ObservedInput.Deserialize(r)); ComputedStrongFingerprint = reader.ReadStrongFingerprint(); IsNewlyPublishedAugmentedWeakFingerprint = reader.ReadBoolean(); AugmentedWeakFingerprint = reader.ReadNullableStruct(r => r.ReadWeakFingerprint()); } else { IsStrongFingerprintHit = false; ObservedInputs = default; ComputedStrongFingerprint = default; IsNewlyPublishedAugmentedWeakFingerprint = false; AugmentedWeakFingerprint = default; } }
private static ObservedPathSet SerializeRoundTripAndAssertEquivalent(PathTable pathTable, ObservedPathSet original, PathExpander pathExpander = null) { using (var mem = new MemoryStream()) { using (var writer = new BuildXLWriter(stream: mem, debug: true, leaveOpen: true, logStats: true)) { original.Serialize(pathTable, writer, pathExpander); } mem.Position = 0; ObservedPathSet roundtrip; using (var reader = new BuildXLReader(stream: mem, debug: true, leaveOpen: true)) { var maybeRoundtrip = ObservedPathSet.TryDeserialize(pathTable, reader, pathExpander); XAssert.IsTrue(maybeRoundtrip.Succeeded, "Failed to deserialize a path set unexpectedly"); roundtrip = maybeRoundtrip.Result; } ObservedPathSetTestUtilities.AssertPathSetsEquivalent(original, roundtrip); return(roundtrip); } }
/// <summary> /// Deserialize result from reader /// </summary> public ExecutionResult Deserialize(BuildXLReader reader, uint workerId) { int minAbsolutePathValue = reader.ReadInt32(); Contract.Assert( minAbsolutePathValue == m_maxSerializableAbsolutePathIndex, "When deserializing for distribution, the minimum absolute path value must match the serialized minimum absolute path value"); var result = (PipResultStatus)reader.ReadByte(); var converged = reader.ReadBoolean(); var numberOfWarnings = reader.ReadInt32Compact(); var weakFingerprint = reader.ReadNullableStruct(r => r.ReadWeakFingerprint()); ProcessPipExecutionPerformance performanceInformation; if (reader.ReadBoolean()) { performanceInformation = ProcessPipExecutionPerformance.Deserialize(reader); // TODO: It looks like this is the wrong class for WorkerId, because the serialized object has always WorkerId == 0. performanceInformation.WorkerId = workerId; } else { performanceInformation = null; } var outputContent = ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .FromWithoutCopy(ReadOutputContent(reader)); var directoryOutputs = ReadOnlyArray <(DirectoryArtifact, ReadOnlyArray <FileArtifact>)> .FromWithoutCopy(ReadDirectoryOutputs(reader)); var mustBeConsideredPerpetuallyDirty = reader.ReadBoolean(); var dynamicallyObservedFiles = reader.ReadReadOnlyArray(ReadAbsolutePath); var dynamicallyProbedFiles = reader.ReadReadOnlyArray(ReadAbsolutePath); var dynamicallyObservedEnumerations = reader.ReadReadOnlyArray(ReadAbsolutePath); var allowedUndeclaredSourceReads = reader.ReadReadOnlySet(ReadAbsolutePath); var absentPathProbesUnderOutputDirectories = reader.ReadReadOnlySet(ReadAbsolutePath); ReportedFileAccess[] fileAccessViolationsNotWhitelisted; ReportedFileAccess[] whitelistedFileAccessViolations; ReportedProcess[] reportedProcesses; ReadReportedProcessesAndFileAccesses(reader, out fileAccessViolationsNotWhitelisted, out whitelistedFileAccessViolations, out reportedProcesses, readPath: m_readPath); var twoPhaseCachingInfo = ReadTwoPhaseCachingInfo(reader); var cacheDescriptor = ReadPipCacheDescriptor(reader); if (cacheDescriptor != null) { cacheDescriptor.OutputContentReplicasMiniBloomFilter = reader.ReadUInt32(); } ObservedPathSet?pathSet = null; if (reader.ReadBoolean()) { var maybePathSet = ObservedPathSet.TryDeserialize(m_executionContext.PathTable, reader, pathReader: ReadAbsolutePath); pathSet = maybePathSet.Succeeded ? (ObservedPathSet?)maybePathSet.Result : null; } CacheLookupPerfInfo cacheLookupCounters = null; if (reader.ReadBoolean()) { cacheLookupCounters = CacheLookupPerfInfo.Deserialize(reader); } if (!result.IndicatesNoOutput()) { // Successful result needs to be changed to not materialized because // the process outputs are not materialized on the local machine. result = PipResultStatus.NotMaterialized; } var pipProperties = ReadPipProperties(reader); var hasUserRetries = reader.ReadBoolean(); var processExecutionResult = ExecutionResult.CreateSealed( result, numberOfWarnings, outputContent, directoryOutputs, performanceInformation, weakFingerprint, fileAccessViolationsNotWhitelisted, whitelistedFileAccessViolations, mustBeConsideredPerpetuallyDirty, dynamicallyObservedFiles, dynamicallyProbedFiles, dynamicallyObservedEnumerations, allowedUndeclaredSourceReads, absentPathProbesUnderOutputDirectories, twoPhaseCachingInfo, cacheDescriptor, converged, pathSet, cacheLookupCounters, pipProperties, hasUserRetries); return(processExecutionResult); }
/// <summary> /// Takes a strong fingerprint and returns the deserialized contents of /// the input assertion list file that corresponds to it /// </summary> /// <param name="sfp">The strong fingerprint to get the input assertion /// list file contents for</param> /// <param name="cacheErrors">Any cache errors that are found will be /// added to this collection</param> /// <returns>Deserialized contents of the input assertion list file /// corresponding to the specified strong fingerprint</returns> private async Task <string> GetInputAssertionListFileContentsAsync(StrongFingerprint sfp, ConcurrentDictionary <CacheError, int> cacheErrors) { // Check for the NoItem if (sfp.CasElement.Equals(CasHash.NoItem)) { return(string.Empty); } // Pin the input assertion list file Possible <string, Failure> possibleString = await m_readOnlySession.PinToCasAsync(sfp.CasElement).ConfigureAwait(false); if (!possibleString.Succeeded) { return(string.Empty); } // Get the stream for the input assertion list file Possible <Stream, Failure> possibleStream = await m_readOnlySession.GetStreamAsync(sfp.CasElement).ConfigureAwait(false); if (!possibleStream.Succeeded) { cacheErrors.TryAdd( new CacheError( CacheErrorType.CasHashError, "The input assertion list for SFP " + sfp.ToString() + " was not found in CAS"), 0); return(string.Empty); } // Read the stream contents while hashing return(await Task.Run(() => { using (var hasher = ContentHashingUtilities.HashInfo.CreateContentHasher()) { using (var hashingStream = hasher.CreateReadHashingStream(possibleStream.Result)) { using (var reader = new BuildXLReader(false, hashingStream, false)) { var maybePathSet = ObservedPathSet.TryDeserialize(s_pathTable, reader); // Check if deserialization was successful if (!maybePathSet.Succeeded) { // Deserialization failed cacheErrors.TryAdd( new CacheError( CacheErrorType.CasHashError, "The input assertion list for SFP " + sfp.ToString() + " could not be deserialized"), 0); return string.Empty; } CasHash newCasHash = new CasHash(hashingStream.GetContentHash()); // Check if the hashes match if (!sfp.CasElement.Equals(newCasHash)) { cacheErrors.TryAdd( new CacheError( CacheErrorType.CasHashError, "The input assertion list for SFP " + sfp.ToString() + " has been altered in the CAS"), 0); return string.Empty; } // Deserialization was successful and file was unaltered StringBuilder fileContents = new StringBuilder(); foreach (ObservedPathEntry entry in maybePathSet.Result.Paths) { fileContents.Append(entry.Path.ToString(s_pathTable)).Append(Environment.NewLine); } return fileContents.ToString(); } } } }).ConfigureAwait(false)); }
/// <summary> /// Check the input list against the regex /// </summary> /// <param name="weak">The weak fingerprint (for logging on failure)</param> /// <param name="casElement">The CasElement of the strong fingerprint</param> /// <param name="hashElement">The hashElement of the strong fingerprint (for logging on failure)</param> /// <param name="urgencyHint">Pass-through</param> /// <param name="activityId">Pass-through activityId</param> /// <returns>false if the check was not performed, true if the checks were performed, failure if the regex checks failed</returns> /// <remarks> /// This will attempt to validate the CAS stored input list against the regex rules /// </remarks> private async Task <Possible <bool, Failure> > CheckInputList(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, UrgencyHint urgencyHint, Guid activityId) { // If we either have no CasHash item or we have no regex to check, just return false // (that we did nothing) if (casElement.Equals(CasHash.NoItem) || ((Cache.MustIncludeRegex == null) && (Cache.MustNotIncludeRegex == null))) { return(false); } // mustInclude start out false if we need to check for mustInclude // Once we get a mustInclude match we no longer need to check. // If we have no mustInclude regex, we set it to true such that // we don't bother checking it bool mustInclude = Cache.MustIncludeRegex == null; // This is just to make a faster check for the MustNotinclude // case. If we have the regex then we must check each entry // but in many cases we don't have the regex so let this be a quick out. bool checkMustNot = Cache.MustNotIncludeRegex != null; // Try to get the observed inputs from the CasHash given var possibleStream = await GetStreamAsync(casElement, urgencyHint, activityId); if (!possibleStream.Succeeded) { // If we could not get a stream to the CasEntery in the fingerprint. return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, "Failed to get stream of CasElement")); } // Deserialize the contents of the path set. using (possibleStream.Result) { PathTable pathTable = new PathTable(); BuildXLReader reader = new BuildXLReader(false, possibleStream.Result, true); var maybePathSet = ObservedPathSet.TryDeserialize(pathTable, reader); if (maybePathSet.Succeeded) { // Deserialization was successful foreach (ObservedPathEntry entry in maybePathSet.Result.Paths) { string filepath = entry.Path.ToString(pathTable); // Have we seen a must-have entry yet? If not check if this is one // that way once we found one we want we stop checking this regex if (!mustInclude) { mustInclude = Cache.MustIncludeRegex.IsMatch(filepath); } // Now, if we are looking for a must not include, we just check for that // and if it matches we fail if (checkMustNot) { if (Cache.MustNotIncludeRegex.IsMatch(filepath)) { return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, string.Format(CultureInfo.InvariantCulture, "Failed due to a MustNotInclude file: {0}", filepath))); } } } } else { return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, "Failed to deserialize observed inputs")); } } if (!mustInclude) { return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, "Failed due to not including at least one MustInclude file")); } return(true); }