/// <summary>
/// Constructor.
/// </summary>
private FingerprintStoreExecutionLogTarget(
LoggingContext loggingContext,
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
FingerprintStore fingerprintStore,
FingerprintStore cacheLookupFingerprintStore,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
CounterCollection <FingerprintStoreCounters> counters,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
m_context = context;
m_pipTable = pipTable;
LoggingContext = loggingContext;
PipContentFingerprinter = pipContentFingerprinter;
ExecutionFingerprintStore = fingerprintStore;
CacheLookupFingerprintStore = cacheLookupFingerprintStore;
// Cache lookup store is per-build state and doesn't need to be garbage collected (vs. execution fignerprint store which is persisted build-over-build)
CacheLookupFingerprintStore?.GarbageCollectCancellationToken.Cancel();
m_pipCacheMissesQueue = new ConcurrentQueue <PipCacheMissInfo>();
Counters = counters;
FingerprintStoreMode = configuration.Logging.FingerprintStoreMode;
m_runtimeCacheMissAnalyzerTask = RuntimeCacheMissAnalyzer.TryCreateAsync(
this,
loggingContext,
context,
configuration,
cache,
graph,
runnablePipPerformance);
Contract.Assume(FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly || CacheLookupFingerprintStore != null, "Unless /storeFingerprints flag is set to /storeFingerprints:ExecutionFingerprintsOnly, the cache lookup FingerprintStore must exist.");
}
public FingerprintTextAnalyzer(AnalysisInput input)
: base(input)
{
m_completedPips = new HashSet <PipId>();
m_contentFingerprinter = null;
m_pathExpander = input.CachedGraph.MountPathExpander;
}
/// <summary>
/// Compute fingerprint for process pip.
/// </summary>
public static ContentFingerprint ComputeFingerprint(this PipContentFingerprinter fingerprinter, Process process)
{
Contract.Requires(fingerprinter != null);
Contract.Requires(process != null);
return(fingerprinter.ComputeWeakFingerprint(process));
}
/// <summary>
/// Constructor.
/// </summary>
private FingerprintStoreExecutionLogTarget(
LoggingContext loggingContext,
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
FingerprintStore fingerprintStore,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
m_context = context;
m_pipTable = pipTable;
PipContentFingerprinter = pipContentFingerprinter;
FingerprintStore = fingerprintStore;
m_pipCacheMissesQueue = new ConcurrentQueue <PipCacheMissInfo>();
m_runtimeCacheMissAnalyzerTask = RuntimeCacheMissAnalyzer.TryCreateAsync(
this,
loggingContext,
context,
configuration,
cache,
graph,
runnablePipPerformance);
}
public PipQueueTestExecutionEnvironment(BuildXLContext context, IConfiguration configuration, PipTable pipTable, string tempDirectory, ISandboxConnection SandboxConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(configuration != null);
Context = context;
LoggingContext = CreateLoggingContextForTest();
Configuration = configuration;
FileContentTable = FileContentTable.CreateNew();
ContentFingerprinter = new PipContentFingerprinter(
context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
ExtraFingerprintSalts.Default(),
pathExpander: PathExpander);
PipTable = pipTable;
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = IpcFactory.GetProvider();
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
Cache = InMemoryCacheFactory.Create();
LocalDiskContentStore = new LocalDiskContentStore(LoggingContext, context.PathTable, FileContentTable, tracker);
m_sandboxConnectionKext = SandboxConnection;
m_expectedWrittenContent = new ConcurrentDictionary <FileArtifact, ContentHash>();
m_wellKnownFiles = new ConcurrentDictionary <FileArtifact, ContentHash>();
m_producers = new ConcurrentDictionary <FileArtifact, Pip>();
m_filesystemView = new TestPipGraphFilesystemView(Context.PathTable);
var fileSystemView = new FileSystemView(Context.PathTable, m_filesystemView, LocalDiskContentStore);
TempCleaner = new TestMoveDeleteCleaner(tempDirectory);
State = new PipExecutionState(
configuration,
cache: new PipTwoPhaseCache(LoggingContext, Cache, context, PathExpander),
unsafeConfiguration: configuration.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: ContentHashingUtilities.CreateRandom(),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: null,
fileSystemView: fileSystemView,
fileContentManager: new FileContentManager(this, new NullOperationTracker()),
directoryMembershipFinterprinterRuleSet: null);
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
/// <summary>
/// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>.
/// </summary>
/// <returns>
/// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to
/// a <see cref="Tracing.FingerprintStore"/> at the provided directory;
/// otherwise, null.
/// </returns>
public static FingerprintStoreExecutionLogTarget Create(
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
string fingerprintStoreDirectory,
LoggingContext loggingContext,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null,
FingerprintStoreTestHooks testHooks = null)
{
var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0);
var possibleStore = FingerprintStore.Open(
fingerprintStoreDirectory,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
testHooks: testHooks);
if (possibleStore.Succeeded)
{
return(new FingerprintStoreExecutionLogTarget(
loggingContext,
context,
pipTable,
pipContentFingerprinter,
possibleStore.Result,
configuration,
cache,
graph,
runnablePipPerformance));
}
else
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleStore.Failure.DescribeIncludingInnerFailures());
}
return(null);
}
public PipContentFingerprinter GetFingerprinter(uint workerId)
{
if (m_contentFingerprinters.Length <= workerId)
{
Array.Resize(ref m_contentFingerprinters, (int)workerId + 1);
}
if (m_contentFingerprinters[workerId] == null)
{
m_contentFingerprinters[workerId] = new PipContentFingerprinter(
CachedGraph.Context.PathTable,
artifact => LookupHashFunction(workerId, artifact),
Salts,
pathExpander: CachedGraph.MountPathExpander,
pipDataLookup: CachedGraph.PipGraph.QueryFileArtifactPipData)
{
FingerprintTextEnabled = true,
};
}
return(m_contentFingerprinters[workerId]);
}
private (Process process, ContentFingerprint contentFingerprint, string fingerprintText) GetFingerprintInfo(PipReference lazyPip)
{
// Make sure the extra event data has set the value properly here.
Contract.Requires(m_fingerprintSalts.HasValue, "m_fingerprintSalts is not set.");
Process pip = (Process)lazyPip.HydratePip();
string fingerprintText;
// This checks for missing content info for pips.
foreach (var dependency in pip.Dependencies)
{
if (!m_fileContentMap.ContainsKey(dependency))
{
return(pip, ContentFingerprint.Zero, "FINGERPRINT CONTAINS UNKNOWN DEPENDENCIES");
}
}
if (m_contentFingerprinter == null)
{
m_contentFingerprinter = new PipContentFingerprinter(
CachedGraph.Context.PathTable,
LookupHash,
m_fingerprintSalts.Value,
pathExpander: m_pathExpander,
pipDataLookup: CachedGraph.PipGraph.QueryFileArtifactPipData)
{
FingerprintTextEnabled = true,
};
}
// TODO: Allow specifying fingerprinting version on the command line
ContentFingerprint fingerprint = m_contentFingerprinter.ComputeWeakFingerprint(
pip,
out fingerprintText);
return(pip, fingerprint, fingerprintText);
}
/// <summary>
/// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>.
/// </summary>
/// <returns>
/// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to
/// a <see cref="Tracing.FingerprintStore"/> at the provided directory;
/// otherwise, null.
/// </returns>
public static FingerprintStoreExecutionLogTarget Create(
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
LoggingContext loggingContext,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
CounterCollection <FingerprintStoreCounters> counters,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null,
FingerprintStoreTestHooks testHooks = null)
{
var fingerprintStorePathString = configuration.Layout.FingerprintStoreDirectory.ToString(context.PathTable);
var cacheLookupFingerprintStorePathString = configuration.Logging.CacheLookupFingerprintStoreLogDirectory.ToString(context.PathTable);
try
{
FileUtilities.CreateDirectoryWithRetry(fingerprintStorePathString);
}
catch (BuildXLException ex)
{
Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message);
throw new BuildXLException("Unable to create fingerprint store directory: ", ex);
}
var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0);
var possibleExecutionStore = FingerprintStore.Open(
fingerprintStorePathString,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
counters: counters,
testHooks: testHooks);
Possible <FingerprintStore> possibleCacheLookupStore = new Failure <string>("No attempt to create a cache lookup fingerprint store yet.");
if (configuration.Logging.FingerprintStoreMode != FingerprintStoreMode.ExecutionFingerprintsOnly)
{
try
{
FileUtilities.CreateDirectoryWithRetry(cacheLookupFingerprintStorePathString);
}
catch (BuildXLException ex)
{
Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message);
throw new BuildXLException("Unable to create fingerprint store directory: ", ex);
}
possibleCacheLookupStore = FingerprintStore.Open(
cacheLookupFingerprintStorePathString,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
counters: counters,
testHooks: testHooks);
}
if (possibleExecutionStore.Succeeded &&
(possibleCacheLookupStore.Succeeded || configuration.Logging.FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly))
{
return(new FingerprintStoreExecutionLogTarget(
loggingContext,
context,
pipTable,
pipContentFingerprinter,
possibleExecutionStore.Result,
possibleCacheLookupStore.Succeeded ? possibleCacheLookupStore.Result : null,
configuration,
cache,
graph,
counters,
runnablePipPerformance));
}
else
{
if (!possibleExecutionStore.Succeeded)
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleExecutionStore.Failure.DescribeIncludingInnerFailures());
}
if (!possibleCacheLookupStore.Succeeded)
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleCacheLookupStore.Failure.DescribeIncludingInnerFailures());
}
}
return(null);
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
IKextConnection sandboxedKextConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(config != null);
LoggingContext = loggingContext;
Context = context;
// Ensure paths visible when debugging
PathTable.DebugPathTable = Context.PathTable;
Configuration = config;
PipTable = pipTable;
PathExpander = semanticPathExpander ?? SemanticPathExpander.Default;
ContentFingerprinter = new PipContentFingerprinter(
Context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null),
pathExpander: PathExpander,
pipDataLookup: pipDataLookup);
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = ipcProvider ?? IpcFactory.GetProvider();
FileContentTable = fileContentTable ?? FileContentTable.CreateNew();
Cache = pipCache;
FileAccessWhitelist = fileAccessWhitelist;
m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories;
m_sandboxedKextConnection = sandboxedKextConnection;
if (Cache == null)
{
Cache = InMemoryCacheFactory.Create(context);
}
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker);
PipGraphView = new TestPipGraphFilesystemView(Context.PathTable);
m_operationTracker = new OperationTracker(loggingContext);
var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore);
var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed;
if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled)
{
preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString());
}
State = new PipExecutionState(
config,
cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: ExecutionLogRecorder,
fileSystemView: fileSystemView,
fileContentManager: GetFileContentManager(),
directoryMembershipFinterprinterRuleSet: null,
unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: preserveOutputsSalt,
serviceManager: new DummyServiceManager());
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
