public static PipStrings CreatePipStrings(Pip pip, CachedGraph cachedGraph)
{
PipStrings result;
if (pip.PipType == PipType.Process)
{
result = new PipStrings(pip as Process, cachedGraph);
}
else if (pip.PipType == PipType.CopyFile)
{
result = new PipStrings(pip as CopyFile, cachedGraph);
}
else if (pip.PipType == PipType.WriteFile)
{
result = new PipStrings(pip as WriteFile, cachedGraph);
}
else
{
result = new PipStrings();
}
result.m_pip = pip;
// Tags
{
var stringTable = cachedGraph.Context.StringTable;
foreach (var tag in pip.Tags)
{
result.m_tags.Add(tag.ToString(stringTable));
}
}
return(result);
}
public SealDirectoryStrings(SealDirectory sealDirectory, CachedGraph cachedGraph)
{
var pathTable = cachedGraph.Context.PathTable;
var stringTable = cachedGraph.Context.StringTable;
m_directoryRoot = sealDirectory.DirectoryRoot.ToString(pathTable).ToLowerInvariant();
{
var patterns = sealDirectory
.Patterns
.Select(pattern => pattern.ToString(stringTable));
foreach (var pattern in patterns)
{
m_patterns.Add(pattern.ToLowerInvariant());
}
}
{
var contents = sealDirectory
.Contents
.Select(fileArtifact => fileArtifact.Path.ToString(pathTable));
foreach (var content in contents)
{
m_contents.Add(content.ToLowerInvariant());
}
}
m_recursive = sealDirectory.Kind == SealDirectoryKind.SourceAllDirectories;
}
private PipStrings(CopyFile copyFilePip, CachedGraph cachedGraph)
{
var pathTable = cachedGraph.Context.PathTable;
m_inputFiles.Add(copyFilePip.Source.Path.ToString(pathTable));
m_outputFiles.Add(copyFilePip.Destination.Path.ToString(pathTable));
}
public void WriteDifferentPips(IEnumerable <DifferentPipPair> differentPips, CachedGraph firstCachedGraph, CachedGraph secondCachedGraph)
{
m_writer.WritePropertyName("DifferentPips");
m_writer.WriteStartArray();
foreach (var pipPair in differentPips)
{
m_writer.WriteStartObject();
m_writer.WritePropertyName("Old");
m_writer.WriteValue(firstCachedGraph.PipTable.GetFormattedSemiStableHash(pipPair.FirstPip));
m_writer.WritePropertyName("New");
m_writer.WriteValue(secondCachedGraph.PipTable.GetFormattedSemiStableHash(pipPair.SecondPip));
{
m_writer.WritePropertyName("Reasons");
m_writer.WriteStartArray();
foreach (var diffReason in pipPair.DiffReasons)
{
m_writer.WriteValue(diffReason.ToString());
}
m_writer.WriteEndArray();
}
m_writer.WriteEndObject();
}
m_writer.WriteEndArray();
}
/// <nodoc />
public static Xldb.Proto.Pip ToPip(this Pip pip, CachedGraph cachedGraph)
{
var xldbPip = new Xldb.Proto.Pip()
{
SemiStableHash = pip.SemiStableHash,
PipId = pip.PipId.Value,
};
foreach (var incomingEdge in cachedGraph.DataflowGraph.GetIncomingEdges(pip.PipId.ToNodeId()))
{
var pipType = cachedGraph.PipTable.HydratePip(incomingEdge.OtherNode.ToPipId(), Pips.PipQueryContext.Explorer).PipType;
if (pipType != PipType.Value && pipType != PipType.HashSourceFile && pipType != PipType.SpecFile && pipType != PipType.Module)
{
xldbPip.IncomingEdges.Add(incomingEdge.OtherNode.Value);
}
}
foreach (var outgoingEdge in cachedGraph.DataflowGraph.GetOutgoingEdges(pip.PipId.ToNodeId()))
{
var pipType = cachedGraph.PipTable.HydratePip(outgoingEdge.OtherNode.ToPipId(), Pips.PipQueryContext.Explorer).PipType;
if (pipType != PipType.Value && pipType != PipType.HashSourceFile && pipType != PipType.SpecFile && pipType != PipType.Module)
{
xldbPip.OutgoingEdges.Add(outgoingEdge.OtherNode.Value);
}
}
return(xldbPip);
}
public bool LoadCacheGraph(string cachedGraphDirectory)
{
if (string.IsNullOrEmpty(cachedGraphDirectory) && string.IsNullOrEmpty(ExecutionLogPath))
{
return(false);
}
// Dummy logger that nothing listens to but is needed for cached graph API
LoggingContext loggingContext = new LoggingContext("BuildXL.Execution.Analyzer");
CachedGraphDirectory = !string.IsNullOrWhiteSpace(cachedGraphDirectory)
? cachedGraphDirectory
: Path.Combine(Path.GetDirectoryName(ExecutionLogPath), Path.GetFileNameWithoutExtension(ExecutionLogPath));
using (ConsoleEventListener listener = new ConsoleEventListener(Events.Log, DateTime.UtcNow,
eventMask: new EventMask(
enabledEvents: null,
disabledEvents: new int[]
{
(int)BuildXL.Engine.Tracing.LogEventId.DeserializedFile, // Don't log anything for success
})
))
{
listener.RegisterEventSource(BuildXL.Engine.ETWLogger.Log);
CachedGraph = CachedGraph.LoadAsync(CachedGraphDirectory, loggingContext, preferLoadingEngineCacheInMemory: true, readStreamProvider: StreamProvider).GetAwaiter().GetResult();
}
if (CachedGraph == null)
{
return(false);
}
return(true);
}
private static Dictionary <AbsolutePath, PipId> GetOutputToPip(CachedGraph cachedGraph)
{
var result = new Dictionary <AbsolutePath, PipId>();
var pips = cachedGraph
.PipTable
.Keys
.Where(pipId =>
{
return(IsRelevantPipType(cachedGraph, pipId));
})
.Select(cachedGraph.PipGraph.GetPipFromPipId);
foreach (var pip in pips)
{
var outputFiles = GetOutputsForPip(pip);
foreach (var path in outputFiles)
{
result.Add(path, pip.PipId);
}
}
return(result);
}
public AnalysisModel(CachedGraph graph)
{
CachedGraph = graph;
ChangedPips = new VisitationTracker(CachedGraph.DataflowGraph);
visitor = new NodeVisitor(graph.DataflowGraph);
LookupHashFunction = LookupHash;
}
protected Analyzer(AnalysisInput input)
: base(input.CachedGraph.PipGraph)
{
Contract.Requires(input.CachedGraph != null);
Input = input;
CachedGraph = input.CachedGraph;
}
public PipDetails(CachedGraph graph, Pip pip)
: base(graph.Context, pip)
{
LongDescription = pip.GetDescription(graph.Context);
Tags.AddRange(pip.Tags.Select(tag => new TagRef(graph.Context, tag)));
Dependencies.AddRange(GetPipRefs(graph.Context, graph.PipGraph.RetrievePipImmediateDependencies(pip)));
Dependents.AddRange(GetPipRefs(graph.Context, graph.PipGraph.RetrievePipImmediateDependents(pip)));
}
public ProcessDetails(CachedGraph graph, Process process)
: base(graph, process)
{
Executable = new FileRef(graph.Context, process.Executable);
Arguments = new PipData(graph.Context, process.Arguments);
EnvironmentVariables.AddRange(process.EnvironmentVariables.Select(envVar => new EnvironmentVariable(graph.Context, envVar)));
WorkingDirectory = new DirectoryRef(graph.Context, process.WorkingDirectory);
UntrackedScopes.AddRange(process.UntrackedScopes.Select(scope => new DirectoryRef(graph.Context, scope)));
UntrackedFiles.AddRange(process.UntrackedPaths.Select(path => new FileRef(graph.Context, FileArtifact.CreateSourceFile(path))));
}
public ConversionModel(CachedGraph oldGraph, CachedGraph newGraph, AnalysisModel oldModel)
{
OldGraph = oldGraph;
OldModel = oldModel;
NewGraph = newGraph;
ConvertedNewModel = new AnalysisModel(NewGraph)
{
LookupHashFunction = ConvertedLookupHash,
};
}
private void WriteUniquePips(IEnumerable <PipId> pipIds, CachedGraph cachedGraph, string propertyName)
{
m_writer.WritePropertyName(propertyName);
m_writer.WriteStartArray();
foreach (var pipId in pipIds)
{
m_writer.WriteValue(cachedGraph.PipTable.GetFormattedSemiStableHash(pipId));
}
m_writer.WriteEndArray();
}
public bool ShouldLoadPip(Process process, CachedGraph cachedGraph)
{
foreach (IPipFilter pipFilter in m_pipFilters)
{
if (!pipFilter.ShouldLoadPip(process, cachedGraph))
{
return(false);
}
}
return(true);
}
public bool ShouldLoadModule(ModulePip module, CachedGraph cachedGraph)
{
foreach (IModuleFilter moduleFilter in m_moduleFilters)
{
if (!moduleFilter.ShouldLoadModule(module, cachedGraph))
{
return(false);
}
}
return(true);
}
private PipStrings(WriteFile writeFilePip, CachedGraph cachedGraph)
{
var pathTable = cachedGraph.Context.PathTable;
m_outputFiles.Add(writeFilePip.Destination.Path.ToString(pathTable));
if (writeFilePip.Contents.IsValid)
{
m_writeFileContents = writeFilePip.Contents.ToString(pathTable);
}
m_writeFileEncoding = writeFilePip.Encoding;
}
private CachedGraph ComputeScaleAndGetCachedGraphFor(IGraph graph)
{
double freq = 1 / GetSizeOfOneStep(graph);
double correction = 0.01;
CachedGraph cachedGraph = new CachedGraph(graph, freq); //TODO decide whether cache graph
Scale = new Scale(
MaxValue: cachedGraph.MaxValue * (1 + correction),
MinValue: cachedGraph.MinValue * (1 + correction)
);
return(cachedGraph);
}
public void DrawGraphAutoscale(IGraph graph)
{
double freq = 1 / GetSizeOfOneStep(graph);
double correction = 0.01;
CachedGraph cachedGraph = new CachedGraph(graph, freq); //TODO decide whether cache graph
Scale = new Scale(
MaxValue: cachedGraph.MaxValue * (1 + correction),
MinValue: cachedGraph.MinValue * (1 + correction)
);
DrawGraph(cachedGraph);
}
protected async Task <CachedGraph> GetCachedGraphAsync(string buildId)
{
var loggingContext = new LoggingContext("Website");
var invocations = new Invocations().GetInvocations(loggingContext);
var thisInvocation = invocations.First(invocation => invocation.SessionId == buildId);
var entry = await Cache.GetOrCreateAsync(
"CachedGraph: =" + buildId,
(newEntry) => CachedGraph.LoadAsync(
Path.Combine(thisInvocation.LogsFolder, "BuildXL"),
loggingContext,
true
));
return(entry);
}
/// <summary>
/// Thread safe GetOrAdd method. We use concurrent dictionaries for backing, therefore no lock is needed.
/// </summary>
/// <param name="pipId">The pip Id to be used when creating a new pip descriptor</param>
/// <param name="pipName">The pip name to be used when creating a new pip descriptor</param>
/// <returns>New PipDescriptor instance that stores the data from the specified pip</returns>
internal PipDescriptor SynchronizedGetOrAdd(Process fullPip, CachedGraph buildGraph,
ExecutionLogLoadOptions loadOptions, ConcurrentHashSet <FileDescriptor> emptyConcurrentHashSetOfFileDescriptor,
ConcurrentHashSet <PipDescriptor> emptyConcurrentHashSetOfPipDescriptor, ConcurrentHashSet <ProcessInstanceDescriptor> emptyConcurrentHashSetOfReportedProcesses,
StringIdEnvVarDictionary emptyStringIDEnvVarDictionary, AbsolutePathConcurrentHashSet emptyAbsolutePathConcurrentHashSet)
{
PipDescriptor newItem = m_pipIdDictionary.GetOrAdd(fullPip.PipId.Value, (p) => { return(new PipDescriptor(fullPip, buildGraph, loadOptions, emptyConcurrentHashSetOfFileDescriptor, emptyConcurrentHashSetOfPipDescriptor, emptyConcurrentHashSetOfReportedProcesses, emptyStringIDEnvVarDictionary, emptyAbsolutePathConcurrentHashSet)); });
IReadOnlyCollection <PipDescriptor> pipList = m_pipNameDictionary.GetOrAdd(fullPip.Provenance.OutputValueSymbol, new ConcurrentHashSet <PipDescriptor>());
// This is pretty ugly: Doing down casting here so we can add elements to our read only collection
// The collection is read only because we do no want to allow the Users of the SDK to change it. Unfortunately the only way .NET allows me to define such dictionary
// is to specify its elements as a IReadOnlyCollection and down cast every time I need to modify it.
// Down casting here is pretty safe though. The collection is only created in this method and we know that it is always a ConcurrentDictionary.
(pipList as ConcurrentHashSet <PipDescriptor>).Add(newItem);
return(newItem);
}
public DependencyAnalyzerOutputWriter(
string outputFilePath,
CachedGraph cachedGraph,
uint graphVersion,
IReadOnlyCollection <AbsolutePath> files,
IReadOnlyCollection <AbsolutePath> dirs,
IReadOnlyList <DependencyAnalyzerPip> pips,
IReadOnlyDictionary <string, string> pathMappings)
{
m_outputFilePath = outputFilePath;
m_cachedGraph = cachedGraph;
m_graphVersion = graphVersion;
m_files = files;
m_dirs = dirs;
m_pips = pips;
m_pathMappings = pathMappings;
}
private bool PipStartsWith(Process process, CachedGraph cachedGraph)
{
bool filterPrefixMatch = false;
string pipName = process.Provenance.OutputValueSymbol.ToString(cachedGraph.Context.SymbolTable);
// Loop trough the filter strings. All these filters are pip name prefixes.
foreach (var filter in m_pipFilters)
{
// Check if the pip name matches the prefix
if (pipName.StartsWith(filter, StringComparison.OrdinalIgnoreCase))
{
// We have a match
filterPrefixMatch = true;
break;
}
}
// We should load the pip when we have a match and the filter IS NOT an exclude filter, or
// we do not have a match and the filter IS an exclude filter.
return(filterPrefixMatch != m_pipExcludeFilter);
}
private bool ModuleStartsWith(ModulePip module, CachedGraph cachedGraph)
{
bool filterPrefixMatch = false;
string moduleName = module.Identity.ToString(cachedGraph.Context.StringTable);
// Loop trough the filter strings. All these filters are module name prefixes.
foreach (var filter in m_moduleFilters)
{
// Check if the module name matches the prefix
if (moduleName.StartsWith(filter, StringComparison.OrdinalIgnoreCase))
{
// We have a match
filterPrefixMatch = true;
break;
}
}
// We should load the module when we have a match and the filter IS NOT an exclude filter, or
// we do not have a match and the filter IS an exclude filter.
return(filterPrefixMatch != m_moduleExcludeFilter);
}
public bool LoadCacheGraph(string cachedGraphDirectory)
{
if (string.IsNullOrEmpty(cachedGraphDirectory) && string.IsNullOrEmpty(ExecutionLogPath))
{
return(false);
}
// Dummy logger that nothing listens to but is needed for cached graph API
LoggingContext loggingContext = new LoggingContext("BuildXL.Execution.Analyzer");
CachedGraphDirectory = !string.IsNullOrWhiteSpace(cachedGraphDirectory)
? cachedGraphDirectory
: Path.Combine(Path.GetDirectoryName(ExecutionLogPath), Path.GetFileNameWithoutExtension(ExecutionLogPath));
CachedGraph = CachedGraph.LoadAsync(CachedGraphDirectory, loggingContext, preferLoadingEngineCacheInMemory: true, readStreamProvider: StreamProvider).GetAwaiter().GetResult();
if (CachedGraph == null)
{
return(false);
}
return(true);
}
/// <summary>
/// Internal constructor
/// </summary>
/// <param name="pipId">The pipId of the pip that this descriptor is assigned to</param>
internal PipDescriptor(Process fullPip, CachedGraph buildGraph, ExecutionLogLoadOptions loadOptions, ConcurrentHashSet <FileDescriptor> emptyConcurrentHashSetOfFileDescriptor, ConcurrentHashSet <PipDescriptor> emptyConcurrentHashSetOfPipDescriptor,
ConcurrentHashSet <ProcessInstanceDescriptor> emptyConcurrentHashSetOfReportedProcesses, StringIdEnvVarDictionary emptyStringIDEnvVarDictionary, AbsolutePathConcurrentHashSet emptyAbsolutePathConcurrentHashSet)
{
Contract.Requires(fullPip != null);
Contract.Requires(buildGraph != null);
// IsInitializedFlag will be set to non 0 when all the pip properties have been set
IsInitializedFlag = 0;
PipExecutionPerformance = null;
m_transitiveDependentPips = Lazy.Create(() => GetTransitiveDependentPips());
m_criticalPathBasedOnNumberOfPipsProducedFromCache = Lazy.Create(() => FindCriticalPathBasedOnNumberOfPipsProducedFromCache());
m_numberOfFilesProducedFromCacheOnCriticalPath = Lazy.Create(() => FindNumberOfFilesProducedFromCacheOnCriticalPath());
m_criticalPathBasedOnExecutionTime = Lazy.Create(() => FindCriticalPathBasedOnExecutionTime());
m_dependencyChainLength = Lazy.Create(() => FindDependencyChainLength());
m_criticalPathLength = Lazy.Create(() => FindCriticalPathLength());
m_buildGraph = buildGraph;
m_fullPip = fullPip;
PipTags = new StringIdConcurrentHashSet(m_buildGraph.Context.StringTable);
foreach (var tag in m_fullPip.Tags)
{
if (tag.IsValid)
{
PipTags.Add(tag);
}
}
if ((loadOptions & ExecutionLogLoadOptions.DoNotLoadOutputFiles) == 0)
{
OutputFilesHashset = new ConcurrentHashSet <FileDescriptor>();
}
else
{
OutputFilesHashset = emptyConcurrentHashSetOfFileDescriptor;
}
if ((loadOptions & ExecutionLogLoadOptions.DoNotLoadSourceFiles) == 0)
{
DependentFilesHashset = new ConcurrentHashSet <FileDescriptor>();
ProbedFilesHashset = new ConcurrentHashSet <FileDescriptor>();
}
else
{
DependentFilesHashset = emptyConcurrentHashSetOfFileDescriptor;
ProbedFilesHashset = emptyConcurrentHashSetOfFileDescriptor;
}
if ((loadOptions & ExecutionLogLoadOptions.LoadObservedInputs) == 0)
{
ObservedInputsHashset = emptyConcurrentHashSetOfFileDescriptor;
}
else
{
ObservedInputsHashset = new ConcurrentHashSet <FileDescriptor>();
}
if ((loadOptions & ExecutionLogLoadOptions.LoadBuildGraph) == 0)
{
AdjacentInNodesHashset = emptyConcurrentHashSetOfPipDescriptor;
AdjacentOutNodesHashset = emptyConcurrentHashSetOfPipDescriptor;
}
else
{
AdjacentInNodesHashset = new ConcurrentHashSet <PipDescriptor>();
AdjacentOutNodesHashset = new ConcurrentHashSet <PipDescriptor>();
}
if ((loadOptions & ExecutionLogLoadOptions.LoadProcessMonitoringData) == 0)
{
ReportedProcessesHashset = emptyConcurrentHashSetOfReportedProcesses;
}
else
{
ReportedProcessesHashset = new ConcurrentHashSet <ProcessInstanceDescriptor>();
}
if ((loadOptions & ExecutionLogLoadOptions.DoNotLoadRarelyUsedPipProperties) == 0)
{
UntrackedPathsHashset = new AbsolutePathConcurrentHashSet(m_buildGraph.Context.PathTable);
UntrackedScopesHashset = new AbsolutePathConcurrentHashSet(m_buildGraph.Context.PathTable);
EnvironmentVariablesDictionary = new StringIdEnvVarDictionary(m_buildGraph.Context);
foreach (var d in m_fullPip.UntrackedPaths)
{
if (d.IsValid)
{
UntrackedPathsHashset.Add(d);
}
}
foreach (var d in m_fullPip.UntrackedScopes)
{
if (d.IsValid)
{
UntrackedScopesHashset.Add(d);
}
}
}
else
{
EnvironmentVariablesDictionary = emptyStringIDEnvVarDictionary;
UntrackedPathsHashset = emptyAbsolutePathConcurrentHashSet;
UntrackedScopesHashset = emptyAbsolutePathConcurrentHashSet;
}
}
public PipExecutionData(CachedGraph cachedGraph, DirectedGraph dataflowGraphOverride = null)
{
CachedGraph = cachedGraph;
DataflowGraph = dataflowGraphOverride ?? CachedGraph.DataflowGraph;
}
public void WriteSecondGraphUniquePips(IEnumerable <PipId> pipIds, CachedGraph cachedGraph)
{
WriteUniquePips(pipIds, cachedGraph, "SecondGraphUniquePips");
}
public bool ShouldLoadModule(ModulePip module, CachedGraph cachedGraph)
{
return(m_moduleFilterFunction.Invoke(module, cachedGraph));
}
private PipStrings(Process processPip, CachedGraph cachedGraph)
{
var pathTable = cachedGraph.Context.PathTable;
var stringTable = cachedGraph.Context.StringTable;
// Input files
{
var inputFiles = processPip
.Dependencies
.Select(x => x.Path.ToString(pathTable));
foreach (var inputFile in inputFiles)
{
m_inputFiles.Add(inputFile);
}
}
// Output files
{
var outputFiles = processPip
.FileOutputs
.Select(x => x.Path.ToString(pathTable));
foreach (var outputFile in outputFiles)
{
m_outputFiles.Add(outputFile);
}
}
// Opaque input dirs
{
var opaqueInputDirs = processPip
.DirectoryDependencies
.Select(x => x.Path.ToString(pathTable));
foreach (var inputDir in opaqueInputDirs)
{
m_opaqueInputDirs.Add(inputDir);
}
}
// Opaque output dirs
{
var opaqueOutputDirs = processPip
.DirectoryOutputs
.Select(x => x.Path.ToString(pathTable));
foreach (var outputDir in opaqueOutputDirs)
{
m_opaqueOutputDirs.Add(outputDir);
}
}
m_executable = processPip.Executable.Path.ToString(pathTable);
m_stdInFile = processPip.StandardInput.File.IsValid ? processPip.StandardInput.File.Path.ToString(pathTable) : null;
m_stdInData = processPip.StandardInput.Data.IsValid ? processPip.StandardInput.Data.ToString(pathTable) : null;
foreach (var envVar in processPip.EnvironmentVariables)
{
var varName = envVar.Name.ToString(stringTable);
var varValue = envVar.Value.IsValid ? envVar.Value.ToString(pathTable) : null;
m_envVars.Add(new KeyValuePair <string, string>(varName, varValue));
}
m_workingDir = processPip.WorkingDirectory.ToString(pathTable);
m_uniqueOutputDir = processPip.UniqueOutputDirectory.ToString(pathTable);
m_tempDir = processPip.TempDirectory.ToString(pathTable);
m_arguments = processPip.Arguments.ToString(pathTable);
// Partially / source seal directory inputs
{
var sealDirectoryInputs = cachedGraph
.PipGraph
.RetrievePipImmediateDependencies(processPip)
.Where(pip => pip.PipType == PipType.SealDirectory)
.Select(pip => pip as SealDirectory)
.Where(sealDirPip => sealDirPip.Kind != SealDirectoryKind.Opaque && sealDirPip.Kind != SealDirectoryKind.SharedOpaque)
.Select(sealDirPip => new SealDirectoryStrings(sealDirPip, cachedGraph));
foreach (var input in sealDirectoryInputs)
{
m_sealDirectoryInputs.Add(input);
}
}
// Untracked paths
{
var untrackedPaths = processPip
.UntrackedPaths
.Select(path => path.ToString(pathTable));
foreach (var path in untrackedPaths)
{
m_untrackedPaths.Add(path);
}
}
// Untracked scopes
{
var untrackedScopes = processPip
.UntrackedScopes
.Select(path => path.ToString(pathTable));
foreach (var scope in untrackedScopes)
{
m_untrackedScopes.Add(scope);
}
}
}
private static bool IsRelevantPipType(CachedGraph cachedGraph, PipId pipId)
{
var pipType = cachedGraph.PipTable.GetPipType(pipId);
return((pipType == PipType.Process) || (pipType == PipType.CopyFile) || (pipType == PipType.WriteFile));
}
