public DirectoryFingerprint?TryComputeDirectoryFingerprint(
AbsolutePath directoryPath,
CacheablePipInfo cachePipInfo,
Func <EnumerationRequest, PathExistence?> tryEnumerateDirectory,
bool cacheableFingerprint,
DirectoryMembershipFingerprinterRule rule,
DirectoryMembershipHashedEventData eventData)
{
Contract.Requires(directoryPath.IsValid);
if (cacheableFingerprint)
{
Contract.Assert(!eventData.IsSearchPath);
string enumerateFilter = eventData.EnumeratePatternRegex ?? RegexDirectoryMembershipFilter.AllowAllRegex;
// Filesystem fingerprints and fingerprints from the full graph may be cached if the filter is AllowAll.
return(m_fingerprints.GetOrAdd((directoryPath, enumerateFilter), Lazy.Create(
() => TryComputeDirectoryFingerprintInternal(
directoryPath,
cachePipInfo,
tryEnumerateDirectory,
rule,
eventData))).Value);
}
eventData.PipId = cachePipInfo.PipId;
return(TryComputeDirectoryFingerprintInternal(
directoryPath,
cachePipInfo,
tryEnumerateDirectory,
rule,
eventData));
}
public DirData(string path, DirectoryMembershipHashedEventData eventData)
{
Path = path;
IsStatic = eventData.IsStatic;
PipId = eventData.PipId;
IsSearchPath = eventData.IsSearchPath;
EnumeratePatternRegex = eventData.EnumeratePatternRegex;
}
public DirectoryFingerprint?TryComputeDirectoryFingerprint(
AbsolutePath directoryPath,
CacheablePipInfo cachePipInfo,
Func <EnumerationRequest, PathExistence?> tryEnumerateDirectory,
bool cacheableFingerprint,
global::BuildXL.Scheduler.DirectoryMembershipFingerprinterRule rule,
DirectoryMembershipHashedEventData eventData)
{
Contract.Requires(directoryPath.IsValid);
return(DirectoryFingerprint.Zero);
}
public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data)
{
var dirData = new DirData(data.Directory.ToString(CachedGraph.Context.PathTable), data);
dirData.Files.Capacity = data.Members.Count;
foreach (var file in data.Members)
{
dirData.Files.Add(file.GetName(CachedGraph.Context.PathTable).ToString(CachedGraph.Context.StringTable));
}
m_dirData.Add(dirData);
}
public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data)
{
if (data.Directory == DirectoryPath)
{
m_writer.WriteLine("Directory Membership");
foreach (var member in data.Members)
{
m_writer.WriteLine(member.ToString(PathTable));
}
m_writer.WriteLine();
}
}
private DirectoryMembershipHashedEventData Convert(DirectoryMembershipHashedEventData data)
{
if (AreGraphsSame)
{
return(data);
}
data.Directory = Convert(data.Directory);
data.PipId = Convert(data.PipId);
data.Members = Convert(data.Members);
return(data);
}
public void DirectoryMembershipHashed(uint workerId, DirectoryMembershipHashedEventData data)
{
var pipId = Convert(data.PipId);
if (pipId.IsValid != data.PipId.IsValid)
{
return;
}
data = Convert(data);
ConvertedNewModel.AddDirectoryData(workerId, data);
}
public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data)
{
if (data.PipId == m_pip.PipId)
{
m_sections.Add(
m_html.CreateBlock(
"Directory Membership Hashed",
m_html.CreateRow("Directory", data.Directory),
m_html.CreateRow("IsSearchPath", data.IsSearchPath),
m_html.CreateRow("IsStatic", data.IsStatic),
m_html.CreateRow("EnumeratePatternRegex", data.EnumeratePatternRegex),
m_html.CreateRow("Members", data.Members)));
}
}
/// <summary>
/// Override event to capture its data and store it in the protobuf
/// </summary>
public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data)
{
var value = data.ToDirectoryMembershipHashedEvent(WorkerID.Value, PathTable, m_nameExpander);
var key = new EventKey
{
EventTypeID = Xldb.Proto.ExecutionEventId.DirectoryMembershipHashed,
PipId = data.PipId.Value,
DirectoryMembershipHashedKey = AbsolutePathToXldbString(data.Directory),
EventSequenceNumber = Interlocked.Increment(ref m_eventSequenceNumber)
};
var keyArr = key.ToByteArray();
var valueArr = value.ToByteArray();
WriteToDb(keyArr, valueArr, XldbDataStore.EventColumnFamilyName);
AddToDbStorageDictionary(DBStoredTypes.DirectoryMembershipHashed, keyArr.Length + valueArr.Length);
}
/// <summary>
/// Adds an entry to the fingerprint store for { directory fingerprint : directory fingerprint inputs }.
/// </summary>
public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data)
{
if (ExecutionFingerprintStore.Disabled)
{
return;
}
using (Counters.StartStopwatch(FingerprintStoreCounters.FingerprintStoreLoggingTime))
{
Counters.IncrementCounter(FingerprintStoreCounters.NumDirectoryMembershipEvents);
var stringContentHash = ContentHashToString(data.DirectoryFingerprint.Hash);
if (!ExecutionFingerprintStore.ContainsContentHash(stringContentHash))
{
Counters.IncrementCounter(FingerprintStoreCounters.NumDirectoryMembershipEntriesPut);
ExecutionFingerprintStore.PutContentHash(stringContentHash, JsonSerialize(data));
}
}
}
/// <nodoc />
public static DirectoryMembershipHashedEvent ToDirectoryMembershipHashedEvent(this DirectoryMembershipHashedEventData data, uint workerID, PathTable pathTable, NameExpander nameExpander)
{
var directoryMembershipEvent = new DirectoryMembershipHashedEvent()
{
WorkerID = workerID,
DirectoryFingerprint = new DirectoryFingerprint()
{
Hash = data.DirectoryFingerprint.Hash.ToContentHash()
},
Directory = data.Directory.ToAbsolutePath(pathTable, nameExpander),
IsStatic = data.IsSearchPath,
IsSearchPath = data.IsSearchPath,
PipID = data.PipId.Value,
EnumeratePatternRegex = data.EnumeratePatternRegex ?? ""
};
directoryMembershipEvent.Members.AddRange(data.Members.Select(member => member.ToAbsolutePath(pathTable, nameExpander)));
return(directoryMembershipEvent);
}
private DirectoryFingerprint?TryComputeDirectoryFingerprintInternal(
AbsolutePath directoryPath,
CacheablePipInfo process,
Func <EnumerationRequest, PathExistence?> tryEnumerateDirectory,
DirectoryMembershipFingerprinterRule rule,
DirectoryMembershipHashedEventData eventData)
{
var expandedDirectoryPath = directoryPath.ToString(m_context.PathTable);
// Log a message if a rule to disable filesystem enumeration is being used
if (rule != null && rule.DisableFilesystemEnumeration)
{
lock (m_pipGraphRuleSpecialCasesLogged)
{
if (m_pipGraphRuleSpecialCasesLogged.Add(directoryPath))
{
Logger.Log.DirectoryFingerprintExercisedRule(m_loggingContext, rule.Name, expandedDirectoryPath);
}
}
}
DirectoryFingerprint result;
PathExistence? existence;
int numMembers = 0;
using (var pooledList = Pools.GetStringList())
{
var directoryMembers = new List <AbsolutePath>();
var fileNames = pooledList.Instance;
// Actually perform the enumeration and compute the fingerprint
Action <AbsolutePath, string> handleEntry = (path, fileName) =>
{
if (rule != null && !rule.DisableFilesystemEnumeration)
{
if (rule.ShouldIgnoreFileWhenEnumerating(fileName))
{
Logger.Log.DirectoryFingerprintExercisedRule(m_loggingContext, rule.Name, path.ToString(m_context.PathTable));
return;
}
}
fileNames.Add(fileName);
directoryMembers.Add(path);
};
existence = tryEnumerateDirectory(new EnumerationRequest(CachedDirectoryContents, directoryPath, process, handleEntry));
if (existence == null)
{
return(null);
}
numMembers = fileNames.Count;
// we sort members here, so they are stored in the same order they are added to a fingerprint in CalculateDirectoryFingerprint
directoryMembers.Sort(m_context.PathTable.ExpandedPathComparer);
eventData.Members = directoryMembers;
result = CalculateDirectoryFingerprint(fileNames);
}
switch (existence)
{
case PathExistence.ExistsAsDirectory:
case PathExistence.ExistsAsFile:
// The file can be a directory symlink or junction, in which case it is classified as a file.
break;
case PathExistence.Nonexistent:
// We return an all-zero fingerprint for a non-existent path.
// This is equivalent to the static-graph case (see ComputeDirectoryFingerprintFromPipGraph)
// since enumerating nonexistent-on-disk path in the path table is valid (though doing so returns an empty result).
Contract.Assume(result.Hash == ContentHashingUtilities.ZeroHash);
break;
default:
throw Contract.AssertFailure("Unhandled PathExistence");
}
eventData.DirectoryFingerprint = result;
m_executionLog?.DirectoryMembershipHashed(eventData);
if (eventData.IsStatic)
{
Logger.Log.DirectoryFingerprintComputedFromGraph(
m_loggingContext,
expandedDirectoryPath,
result.ToString(),
numMembers,
process.Description);
}
else
{
Logger.Log.DirectoryFingerprintComputedFromFilesystem(
m_loggingContext,
expandedDirectoryPath,
result.ToString(),
numMembers);
}
return(result);
}
/// <inheritdoc />
public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data)
{
m_model.AddDirectoryData(CurrentEventWorkerId, data);
}
/// <inheritdoc />
public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data)
{
m_model.DirectoryData[(CurrentEventWorkerId, data.Directory, data.PipId, data.EnumeratePatternRegex)] = data;
/// <nodoc />
public static Xldb.DirectoryMembershipHashedEvent ToDirectoryMembershipHashedEvent(this DirectoryMembershipHashedEventData data, uint workerID, PathTable pathTable)
{
var Uuid = Guid.NewGuid().ToString();
var directoryMembershipEvent = new Xldb.DirectoryMembershipHashedEvent()
{
UUID = Uuid,
WorkerID = workerID,
DirectoryFingerprint = new Xldb.DirectoryFingerprint()
{
Hash = new Xldb.ContentHash()
{
Value = data.DirectoryFingerprint.Hash.ToString()
}
},
Directory = data.Directory.ToAbsolutePath(pathTable),
IsStatic = data.IsSearchPath,
IsSearchPath = data.IsSearchPath,
PipID = data.PipId.Value,
EnumeratePatternRegex = data.EnumeratePatternRegex ?? ""
};
directoryMembershipEvent.Members.AddRange(data.Members.Select(member => member.ToAbsolutePath(pathTable)));
return(directoryMembershipEvent);
}
/// <inheritdoc />
public virtual void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data)
{
ReportUnhandledEvent(data);
}
public override void DirectoryMembershipHashed(DirectoryMembershipHashedEventData data)
{
m_dirData.Add(data.Directory, data);
}
