/// <summary>
/// Add a pip, when you are sure it's not already in the table.
/// </summary>
public PipId Add(long semiStableHash, string name, PipType pipType)
{
PipEntry entry = new PipEntry(
semiStableHash,
NameTableBuilder.GetOrAdd(name),
pipType);
return(((PipTable)ValueTable).Add(entry));
}
private PipEntry CreateEntry(PipId pipId)
{
var pip = GetPip(pipId);
var entry = new PipEntry()
{
PipId = pipId,
PipType = pip.PipType,
Process = pip as Process,
SemistableHash = pip.SemiStableHash,
Identifier = $"{pip.FormattedSemiStableHash} [{pip.PipType}]",
SpecFile = pip.Provenance?.Token.Path ?? AbsolutePath.Invalid,
};
if (entry.SpecFile.IsValid)
{
entry.SpecFileName = entry.SpecFile.GetName(PathTable).ToString(StringTable);
}
return(entry);
}
private bool TryResolve(FileArtifact file, PipEntry consumer, out FileArtifact resolvedFile)
{
resolvedFile = default;
bool result = false;
while (file.IsOutputFile)
{
if (m_copiedFilesByTarget.TryGetValue(file, out var source))
{
resolvedFile = source;
file = source;
result = true;
}
else
{
break;
}
}
return(result);
}
public void PackedExecution_can_store_pips()
{
PackedExecution packedExecution = new PackedExecution();
PackedExecution.Builder packedExecutionBuilder = new PackedExecution.Builder(packedExecution);
long hash = 1;
string name = "ShellCommon.Shell.ShellCommon.Shell.Merged.Winmetadata";
PipId id = packedExecutionBuilder.PipTableBuilder.Add(hash, name, PipType.Process);
XAssert.AreEqual(0, packedExecution.DirectoryTable.Count);
XAssert.AreEqual(0, packedExecution.FileTable.Count);
XAssert.AreEqual(0, packedExecution.PathTable.Count);
XAssert.AreEqual(1, packedExecution.PipTable.Count);
XAssert.AreEqual(4, packedExecution.StringTable.Count);
XAssert.AreEqual(0, packedExecution.WorkerTable.Count);
PipEntry entry = packedExecution.PipTable[id];
XAssert.AreEqual(hash, entry.SemiStableHash);
XAssert.AreEqual(name, packedExecution.PipTable.PipNameTable.GetText(entry.Name));
}
public void ProcessFingerprintComputed(ProcessFingerprintComputationEventData data)
{
if (data.Kind != FingerprintComputationKind.Execution)
{
return;
}
m_consumer = m_analyzer.GetEntry(data.PipId);
m_consumedFilesByPath.Clear();
m_dependencyConsumedFileIndex.Clear();
m_dependencyConsumedFileEndIndex.Clear();
m_dependencies.Clear();
m_builder.Clear();
m_directoryDependenciesFilterMap.Clear();
m_directoryHasSources.Clear();
var computation = data.StrongFingerprintComputations[0];
var pip = (Process)m_analyzer.GetPip(data.PipId);
PipArtifacts.ForEachInput(pip, input =>
{
if (input.IsFile)
{
AddConsumedFile(input.FileArtifact, DirectoryArtifact.Invalid, ContentFlag.Static | ContentFlag.Consumed);
}
else
{
foreach (var file in m_analyzer.GetContents(input.DirectoryArtifact))
{
if (file.IsSourceFile)
{
m_directoryHasSources.Add(input.DirectoryArtifact);
}
AddConsumedFile(file, input.DirectoryArtifact, ContentFlag.Dynamic);
}
}
return(true);
}, includeLazyInputs: false);
foreach (var input in computation.ObservedInputs)
{
var flag = (ContentFlag)((int)ContentFlag.AbsentPathProbe << (int)input.Type) | ContentFlag.Consumed;
if (input.Type == ObservedInputType.FileContentRead || input.Type == ObservedInputType.ExistingFileProbe)
{
if (m_consumedFilesByPath.TryGetValue(input.Path, out var file))
{
file.AddFlag(ContentFlag.Consumed);
if (file.SourceFile != null)
{
file.SourceFile.AddFlag(ContentFlag.Consumed);
}
if (file.FinalFile != null)
{
file.FinalFile.AddFlag(ContentFlag.Consumed);
}
}
else
{
AddConsumedFile(FileArtifact.CreateSourceFile(input.Path), m_analyzer.PipGraph.TryGetSealSourceAncestor(input.Path), flag | ContentFlag.Unknown);
}
}
else if (m_analyzer.AllAccesses)
{
AddConsumedFile(FileArtifact.CreateSourceFile(input.Path), m_analyzer.PipGraph.TryGetSealSourceAncestor(input.Path), flag);
}
}
var entry = m_consumer;
// Sort file dependencies for consistent output
entry.FileDependencies.Sort(s_fileReferenceComparer);
foreach (var fileDependency in entry.FileDependencies)
{
if (fileDependency.Producer != null)
{
var reference = entry.PipDependencies.GetOrAdd(fileDependency.Producer.PipId, p => new PipReference());
if (reference.Pip == null)
{
reference.Pip = m_analyzer.GetEntry(fileDependency.Producer.PipId);
}
reference.Flags |= fileDependency.ConsumedFile.Flags;
}
}
string describe(PipEntry pe)
{
return($"{pe.SpecFileName}-{m_analyzer.GetDescription(m_analyzer.GetPip(pe.PipId))}");
}
m_builder.AppendLine(describe(entry));
foreach (var fileDependency in entry.FileDependencies)
{
if (fileDependency.Producer != null &&
fileDependency.ConsumedFile.File.Artifact.IsOutputFile)
{
var pipId = fileDependency.Producer.PipId;
var pipReference = entry.PipDependencies[pipId];
var directory = fileDependency.Directory?.Directory ?? DirectoryArtifact.Invalid;
if (m_dependencies.Add((pipId, directory)))
{
if (pipReference.HasFlag(ContentFlag.Consumed))
{
m_directoryDependenciesFilterMap[directory] = true;
m_builder.AppendLine($"{entry.Identifier} -> Retaining pip dependency on '{describe(pipReference.Pip)}' (declared via directory '{ToString(fileDependency.Directory)}') (consumes '{ToString(fileDependency.ConsumedFile.File.Artifact)}')");
}
else
{
m_directoryDependenciesFilterMap.TryAdd(directory, false);
m_builder.AppendLine($"{entry.Identifier} -> Removing pip dependency on '{describe(pipReference.Pip)}' (declared via directory '{ToString(fileDependency.Directory)}')");
}
}
}
}
var trimmedDirectoryDependencies = new List <DirectoryArtifact>();
foreach (var d in entry.Process.DirectoryDependencies)
{
if (m_directoryDependenciesFilterMap.TryGetValue(d, out var shouldInclude))
{
if (shouldInclude)
{
m_builder.AppendLine($"{entry.Identifier} -> Retaining directory dependency on '{ToString(d)}' (used)");
}
else if (m_directoryHasSources.Contains(d))
{
m_builder.AppendLine($"{entry.Identifier} -> Retaining directory dependency on '{ToString(d)}' (has sources)");
}
else
{
m_builder.AppendLine($"{entry.Identifier} -> Removing directory dependency on '{ToString(d)}'");
continue;
}
}
else
{
var sealId = m_analyzer.PipGraph.GetSealedDirectoryNode(d).ToPipId();
if (!m_directoryHasSources.Contains(d) && !m_analyzer.PipTable.GetSealDirectoryKind(sealId).IsSourceSeal())
{
m_builder.AppendLine($"{entry.Identifier} -> Removing directory dependency on '{ToString(d)}' (unused output directory)");
continue;
}
}
entry.PipDependencies.TryAdd(m_analyzer.PipGraph.GetSealedDirectoryNode(d).ToPipId(), default);
trimmedDirectoryDependencies.Add(d);
}
// Update directory dependencies which trimmed directory dependencies to allow writing
// a pip into the serialized pip table that can run without the unnecessary dependencies
entry.Process.UnsafeUpdateDirectoryDependencies(trimmedDirectoryDependencies.ToReadOnlyArray());
m_builder.AppendLine();
// Update the graph
var modifiedGraph = m_analyzer.m_mutableGraph;
using (var scope = modifiedGraph.AcquireExclusiveIncomingEdgeScope(entry.PipId.ToNodeId()))
{
foreach (var dependency in entry.PipDependencies)
{
if (dependency.Value == null || dependency.Value.HasFlag(ContentFlag.Consumed))
{
scope.AddEdge(dependency.Key.ToNodeId());
}
}
entry.AddedEdges = true;
}
if (m_analyzer.SemiStableHashes.Contains(entry.SemistableHash))
{
using (var writer = new StreamWriter(Path.Combine(m_analyzer.OutputFilePath,
$"{GetFileName(entry.SpecFile)}_Pip{pip.FormattedSemiStableHash}.csv")))
{
var table = new DisplayTable <Columns>(" , ");
foreach (var dependency in entry.FileDependencies)
{
table.NextRow();
table.Set(Columns.Path, ToString(dependency.ConsumedFile.File.Artifact.Path));
table.Set(Columns.RwCount, dependency.ConsumedFile.File.Artifact.RewriteCount.ToString());
table.Set(Columns.Flags, dependency.ConsumedFile.Flags.ToString());
table.Set(Columns.Producer, dependency.Producer?.Identifier);
table.Set(Columns.ProducerSpec, GetFileName(dependency.Producer?.SpecFile ?? AbsolutePath.Invalid));
table.Set(Columns.Dir, ToString(dependency.Directory));
table.Set(Columns.DirId, dependency.Directory?.Id);
table.Set(Columns.DirSsh, dependency.Directory?.SemistableHash);
}
table.Write(writer);
}
}
if (m_builder.Length != 0)
{
m_analyzer.Write(m_builder);
}
}
