public DirectoryFingerprint?TryComputeDirectoryFingerprint( AbsolutePath directoryPath, CacheablePipInfo cachePipInfo, Func <EnumerationRequest, PathExistence?> tryEnumerateDirectory, bool cacheableFingerprint, DirectoryMembershipFingerprinterRule rule, DirectoryMembershipHashedEventData eventData) { Contract.Requires(directoryPath.IsValid); if (cacheableFingerprint) { Contract.Assert(!eventData.IsSearchPath); string enumerateFilter = eventData.EnumeratePatternRegex ?? RegexDirectoryMembershipFilter.AllowAllRegex; // Filesystem fingerprints and fingerprints from the full graph may be cached if the filter is AllowAll. return(m_fingerprints.GetOrAdd((directoryPath, enumerateFilter), Lazy.Create( () => TryComputeDirectoryFingerprintInternal( directoryPath, cachePipInfo, tryEnumerateDirectory, rule, eventData))).Value); } eventData.PipId = cachePipInfo.PipId; return(TryComputeDirectoryFingerprintInternal( directoryPath, cachePipInfo, tryEnumerateDirectory, rule, eventData)); }
public DirData(string path, DirectoryMembershipHashedEventData eventData) { Path = path; IsStatic = eventData.IsStatic; PipId = eventData.PipId; IsSearchPath = eventData.IsSearchPath; EnumeratePatternRegex = eventData.EnumeratePatternRegex; }
public DirectoryFingerprint?TryComputeDirectoryFingerprint( AbsolutePath directoryPath, CacheablePipInfo cachePipInfo, Func <EnumerationRequest, PathExistence?> tryEnumerateDirectory, bool cacheableFingerprint, global::BuildXL.Scheduler.DirectoryMembershipFingerprinterRule rule, DirectoryMembershipHashedEventData eventData) { Contract.Requires(directoryPath.IsValid); return(DirectoryFingerprint.Zero); }
public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data) { var dirData = new DirData(data.Directory.ToString(CachedGraph.Context.PathTable), data); dirData.Files.Capacity = data.Members.Count; foreach (var file in data.Members) { dirData.Files.Add(file.GetName(CachedGraph.Context.PathTable).ToString(CachedGraph.Context.StringTable)); } m_dirData.Add(dirData); }
public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data) { if (data.Directory == DirectoryPath) { m_writer.WriteLine("Directory Membership"); foreach (var member in data.Members) { m_writer.WriteLine(member.ToString(PathTable)); } m_writer.WriteLine(); } }
private DirectoryMembershipHashedEventData Convert(DirectoryMembershipHashedEventData data) { if (AreGraphsSame) { return(data); } data.Directory = Convert(data.Directory); data.PipId = Convert(data.PipId); data.Members = Convert(data.Members); return(data); }
public void DirectoryMembershipHashed(uint workerId, DirectoryMembershipHashedEventData data) { var pipId = Convert(data.PipId); if (pipId.IsValid != data.PipId.IsValid) { return; } data = Convert(data); ConvertedNewModel.AddDirectoryData(workerId, data); }
public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data) { if (data.PipId == m_pip.PipId) { m_sections.Add( m_html.CreateBlock( "Directory Membership Hashed", m_html.CreateRow("Directory", data.Directory), m_html.CreateRow("IsSearchPath", data.IsSearchPath), m_html.CreateRow("IsStatic", data.IsStatic), m_html.CreateRow("EnumeratePatternRegex", data.EnumeratePatternRegex), m_html.CreateRow("Members", data.Members))); } }
/// <summary> /// Override event to capture its data and store it in the protobuf /// </summary> public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data) { var value = data.ToDirectoryMembershipHashedEvent(WorkerID.Value, PathTable, m_nameExpander); var key = new EventKey { EventTypeID = Xldb.Proto.ExecutionEventId.DirectoryMembershipHashed, PipId = data.PipId.Value, DirectoryMembershipHashedKey = AbsolutePathToXldbString(data.Directory), EventSequenceNumber = Interlocked.Increment(ref m_eventSequenceNumber) }; var keyArr = key.ToByteArray(); var valueArr = value.ToByteArray(); WriteToDb(keyArr, valueArr, XldbDataStore.EventColumnFamilyName); AddToDbStorageDictionary(DBStoredTypes.DirectoryMembershipHashed, keyArr.Length + valueArr.Length); }
/// <summary> /// Adds an entry to the fingerprint store for { directory fingerprint : directory fingerprint inputs }. /// </summary> public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data) { if (ExecutionFingerprintStore.Disabled) { return; } using (Counters.StartStopwatch(FingerprintStoreCounters.FingerprintStoreLoggingTime)) { Counters.IncrementCounter(FingerprintStoreCounters.NumDirectoryMembershipEvents); var stringContentHash = ContentHashToString(data.DirectoryFingerprint.Hash); if (!ExecutionFingerprintStore.ContainsContentHash(stringContentHash)) { Counters.IncrementCounter(FingerprintStoreCounters.NumDirectoryMembershipEntriesPut); ExecutionFingerprintStore.PutContentHash(stringContentHash, JsonSerialize(data)); } } }
/// <nodoc /> public static DirectoryMembershipHashedEvent ToDirectoryMembershipHashedEvent(this DirectoryMembershipHashedEventData data, uint workerID, PathTable pathTable, NameExpander nameExpander) { var directoryMembershipEvent = new DirectoryMembershipHashedEvent() { WorkerID = workerID, DirectoryFingerprint = new DirectoryFingerprint() { Hash = data.DirectoryFingerprint.Hash.ToContentHash() }, Directory = data.Directory.ToAbsolutePath(pathTable, nameExpander), IsStatic = data.IsSearchPath, IsSearchPath = data.IsSearchPath, PipID = data.PipId.Value, EnumeratePatternRegex = data.EnumeratePatternRegex ?? "" }; directoryMembershipEvent.Members.AddRange(data.Members.Select(member => member.ToAbsolutePath(pathTable, nameExpander))); return(directoryMembershipEvent); }
private DirectoryFingerprint?TryComputeDirectoryFingerprintInternal( AbsolutePath directoryPath, CacheablePipInfo process, Func <EnumerationRequest, PathExistence?> tryEnumerateDirectory, DirectoryMembershipFingerprinterRule rule, DirectoryMembershipHashedEventData eventData) { var expandedDirectoryPath = directoryPath.ToString(m_context.PathTable); // Log a message if a rule to disable filesystem enumeration is being used if (rule != null && rule.DisableFilesystemEnumeration) { lock (m_pipGraphRuleSpecialCasesLogged) { if (m_pipGraphRuleSpecialCasesLogged.Add(directoryPath)) { Logger.Log.DirectoryFingerprintExercisedRule(m_loggingContext, rule.Name, expandedDirectoryPath); } } } DirectoryFingerprint result; PathExistence? existence; int numMembers = 0; using (var pooledList = Pools.GetStringList()) { var directoryMembers = new List <AbsolutePath>(); var fileNames = pooledList.Instance; // Actually perform the enumeration and compute the fingerprint Action <AbsolutePath, string> handleEntry = (path, fileName) => { if (rule != null && !rule.DisableFilesystemEnumeration) { if (rule.ShouldIgnoreFileWhenEnumerating(fileName)) { Logger.Log.DirectoryFingerprintExercisedRule(m_loggingContext, rule.Name, path.ToString(m_context.PathTable)); return; } } fileNames.Add(fileName); directoryMembers.Add(path); }; existence = tryEnumerateDirectory(new EnumerationRequest(CachedDirectoryContents, directoryPath, process, handleEntry)); if (existence == null) { return(null); } numMembers = fileNames.Count; // we sort members here, so they are stored in the same order they are added to a fingerprint in CalculateDirectoryFingerprint directoryMembers.Sort(m_context.PathTable.ExpandedPathComparer); eventData.Members = directoryMembers; result = CalculateDirectoryFingerprint(fileNames); } switch (existence) { case PathExistence.ExistsAsDirectory: case PathExistence.ExistsAsFile: // The file can be a directory symlink or junction, in which case it is classified as a file. break; case PathExistence.Nonexistent: // We return an all-zero fingerprint for a non-existent path. // This is equivalent to the static-graph case (see ComputeDirectoryFingerprintFromPipGraph) // since enumerating nonexistent-on-disk path in the path table is valid (though doing so returns an empty result). Contract.Assume(result.Hash == ContentHashingUtilities.ZeroHash); break; default: throw Contract.AssertFailure("Unhandled PathExistence"); } eventData.DirectoryFingerprint = result; m_executionLog?.DirectoryMembershipHashed(eventData); if (eventData.IsStatic) { Logger.Log.DirectoryFingerprintComputedFromGraph( m_loggingContext, expandedDirectoryPath, result.ToString(), numMembers, process.Description); } else { Logger.Log.DirectoryFingerprintComputedFromFilesystem( m_loggingContext, expandedDirectoryPath, result.ToString(), numMembers); } return(result); }
/// <inheritdoc /> public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data) { m_model.AddDirectoryData(CurrentEventWorkerId, data); }
/// <inheritdoc /> public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data) { m_model.DirectoryData[(CurrentEventWorkerId, data.Directory, data.PipId, data.EnumeratePatternRegex)] = data;
/// <nodoc /> public static Xldb.DirectoryMembershipHashedEvent ToDirectoryMembershipHashedEvent(this DirectoryMembershipHashedEventData data, uint workerID, PathTable pathTable) { var Uuid = Guid.NewGuid().ToString(); var directoryMembershipEvent = new Xldb.DirectoryMembershipHashedEvent() { UUID = Uuid, WorkerID = workerID, DirectoryFingerprint = new Xldb.DirectoryFingerprint() { Hash = new Xldb.ContentHash() { Value = data.DirectoryFingerprint.Hash.ToString() } }, Directory = data.Directory.ToAbsolutePath(pathTable), IsStatic = data.IsSearchPath, IsSearchPath = data.IsSearchPath, PipID = data.PipId.Value, EnumeratePatternRegex = data.EnumeratePatternRegex ?? "" }; directoryMembershipEvent.Members.AddRange(data.Members.Select(member => member.ToAbsolutePath(pathTable))); return(directoryMembershipEvent); }
/// <inheritdoc /> public virtual void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data) { ReportUnhandledEvent(data); }
public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data) { m_dirData.Add(data.Directory, data); }