/// <summary>
/// Creates an instance of <see cref="PipGraphFragmentBuilder"/>.
/// </summary>
public PipGraphFragmentBuilder(
PipExecutionContext pipExecutionContext,
IConfiguration configuration,
PathExpander pathExpander)
{
Contract.Requires(pipExecutionContext != null);
Configuration = configuration;
m_pipExecutionContext = pipExecutionContext;
m_lazyApiServerMoniker = configuration.Schedule.UseFixedApiServerMoniker
? Lazy.Create(() => IpcFactory.GetFixedMoniker())
: Lazy.Create(() => IpcFactory.GetProvider().CreateNewMoniker());
SealDirectoryTable = new SealedDirectoryTable(m_pipExecutionContext.PathTable);
if (configuration.Schedule.ComputePipStaticFingerprints)
{
var extraFingerprintSalts = new ExtraFingerprintSalts(
configuration,
PipFingerprintingVersion.TwoPhaseV2,
configuration.Cache.CacheSalt,
new DirectoryMembershipFingerprinterRuleSet(configuration, pipExecutionContext.StringTable).ComputeSearchPathToolsHash());
m_pipStaticFingerprinter = new PipStaticFingerprinter(
pipExecutionContext.PathTable,
sealDirectoryFingerprintLookup: null,
directoryProducerFingerprintLookup: null,
extraFingerprintSalts: extraFingerprintSalts,
pathExpander: pathExpander)
{
FingerprintTextEnabled = configuration.Schedule.LogPipStaticFingerprintTexts
};
}
}
private PipContentFingerprinter CreateFingerprinter(FileArtifact executable)
{
return(new PipContentFingerprinter(
m_context.PathTable,
PipFingerprinterTests.GetContentHashLookup(executable),
ExtraFingerprintSalts.Default())
{
FingerprintTextEnabled = true
});
}
public PipQueueTestExecutionEnvironment(BuildXLContext context, IConfiguration configuration, PipTable pipTable, string tempDirectory, ISandboxConnection SandboxConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(configuration != null);
Context = context;
LoggingContext = CreateLoggingContextForTest();
Configuration = configuration;
FileContentTable = FileContentTable.CreateNew();
ContentFingerprinter = new PipContentFingerprinter(
context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
ExtraFingerprintSalts.Default(),
pathExpander: PathExpander);
PipTable = pipTable;
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = IpcFactory.GetProvider();
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
Cache = InMemoryCacheFactory.Create();
LocalDiskContentStore = new LocalDiskContentStore(LoggingContext, context.PathTable, FileContentTable, tracker);
m_sandboxConnectionKext = SandboxConnection;
m_expectedWrittenContent = new ConcurrentDictionary <FileArtifact, ContentHash>();
m_wellKnownFiles = new ConcurrentDictionary <FileArtifact, ContentHash>();
m_producers = new ConcurrentDictionary <FileArtifact, Pip>();
m_filesystemView = new TestPipGraphFilesystemView(Context.PathTable);
var fileSystemView = new FileSystemView(Context.PathTable, m_filesystemView, LocalDiskContentStore);
TempCleaner = new TestMoveDeleteCleaner(tempDirectory);
State = new PipExecutionState(
configuration,
cache: new PipTwoPhaseCache(LoggingContext, Cache, context, PathExpander),
unsafeConfiguration: configuration.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: ContentHashingUtilities.CreateRandom(),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: null,
fileSystemView: fileSystemView,
fileContentManager: new FileContentManager(this, new NullOperationTracker()),
directoryMembershipFinterprinterRuleSet: null);
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
/// <summary>
/// Creates an instance of <see cref="GraphFragmentBuilder"/>.
/// </summary>
public GraphFragmentBuilderTopSort(LoggingContext loggingContext, PipExecutionContext pipExecutionContext, IConfiguration configuration, PathExpander pathExpander) : base(loggingContext, pipExecutionContext, configuration)
{
if (configuration.Schedule.ComputePipStaticFingerprints)
{
var extraFingerprintSalts = new ExtraFingerprintSalts(
configuration,
PipFingerprintingVersion.TwoPhaseV2,
configuration.Cache.CacheSalt,
new DirectoryMembershipFingerprinterRuleSet(configuration, pipExecutionContext.StringTable).ComputeSearchPathToolsHash());
m_pipStaticFingerprinter = new PipStaticFingerprinter(
pipExecutionContext.PathTable,
GetSealDirectoryFingerprint,
GetDirectoryProducerFingerprint,
extraFingerprintSalts,
pathExpander)
{
FingerprintTextEnabled = configuration.Schedule.LogPipStaticFingerprintTexts
};
}
}
/// <summary>
/// Computes a fingerprint for looking up fingerprint store
/// </summary>
private static ContentFingerprint ComputeFingerprint(
string key,
IConfiguration configuration)
{
var extraFingerprintSalts = new ExtraFingerprintSalts(
configuration,
configuration.Cache.CacheSalt,
null);
using (var hasher = new BasicHashingHelper(recordFingerprintString: false))
{
hasher.Add("Type", "FingerprintStoreFingerprint");
hasher.Add("FormatVersion", FingerprintStore.FormatVersion.Version.ToString());
hasher.Add("LookupVersion", FingerprintStoreLookupVersion);
hasher.Add("Key", key);
hasher.Add("FingerprintSalt", extraFingerprintSalts.FingerprintSalt);
var fingerprint = new ContentFingerprint(hasher.GenerateHash());
return(fingerprint);
}
}
private static Optional <CompositeGraphFingerprint> GenerateHash(
LoggingContext loggingContext,
IStartupConfiguration startUpConfig,
IConfiguration config,
PathTable pathTable,
IEvaluationFilter partialEvaluationData,
FileContentTable fileContentTable,
string commitId,
EngineTestHooksData testHooks)
{
ILayoutConfiguration layout = config.Layout;
ILoggingConfiguration logging = config.Logging;
var fingerprintTextElements = new List <(string, string)>();
using (var hasher = new CoreHashingHelper(recordFingerprintString: true))
{
CompositeGraphFingerprint fingerprint = CompositeGraphFingerprint.Zero;
AddInt(hasher, "version", GraphFingerprintVersion);
using (var qualifierHasher = new CoreHashingHelper(recordFingerprintString: false))
{
foreach (string qualifier in startUpConfig.QualifierIdentifiers.OrderBy(q => q))
{
AddText(qualifierHasher, "qualifier", qualifier);
}
fingerprint.QualifierHash = qualifierHasher.GenerateHash();
AddFingerprint(hasher, "Qualifiers", fingerprint.QualifierHash);
}
if (partialEvaluationData != null && partialEvaluationData.CanPerformPartialEvaluation)
{
using (var evaluationFilterHasher = new CoreHashingHelper(recordFingerprintString: false))
{
foreach (string value in partialEvaluationData.ValueNamesToResolveAsStrings.OrderBy(v => v))
{
AddText(evaluationFilterHasher, "valueName", value);
}
foreach (string value in partialEvaluationData.ValueDefinitionRootsToResolveAsStrings.OrderBy(v => v))
{
AddText(evaluationFilterHasher, "valuePath", value);
}
foreach (string value in partialEvaluationData.ModulesToResolveAsStrings.OrderBy(v => v))
{
AddText(evaluationFilterHasher, "moduleName", value);
}
fingerprint.FilterHash = evaluationFilterHasher.GenerateHash();
AddFingerprint(hasher, "Values", fingerprint.FilterHash);
}
}
// These paths get embedded in the result of evaluation. So if any change we must re-evaluate
AddText(hasher, "ObjectDirectoryPath", layout.ObjectDirectory.IsValid ? layout.ObjectDirectory.ToString(pathTable) : "::null::");
AddText(hasher, "TempDirectoryPath", layout.TempDirectory.IsValid ? layout.TempDirectory.ToString(pathTable) : "::null::");
AddText(hasher, "SourceDirectoryPath", layout.SourceDirectory.IsValid ? layout.SourceDirectory.ToString(pathTable) : "::null::");
// All paths in the graph are relative to 'substTarget' (hence, 'substTarget' must be a part of the fingerprint, but 'substSource' need not be).
AddText(hasher, "substTarget", logging.SubstTarget.IsValid ? logging.SubstTarget.ToString(pathTable) : "::null::");
AddText(hasher, "IsCompressed", config.Engine.CompressGraphFiles ? "true" : "false");
AddText(hasher, "IsSkipHashSourceFile", config.Schedule.SkipHashSourceFile ? "true" : "false");
// Pip static fingerprints are not always computed because computing them slows down the graph construction by 10%-13%.
// Thus, the pip graph may and may not contain pip static fingerprints. To avoid unexpected result due to graph cache hit,
// we temporarily add the option for computing pip static fingerprints as part of our graph fingerprint until the fingerprints
// are always computed; see Task 1291638.
AddText(hasher, "ComputePipStaticFingerprints", config.Schedule.ComputePipStaticFingerprints.ToString());
if (config.Schedule.ComputePipStaticFingerprints)
{
// Pip static fingerprints are part of the graph and include the extra fingerprint salts.
// Thus, when pip static fingerprints are computed, any change to the salt will invalidate the graph because
// the pip static fingerprints will no longer be valid. Reusing the graph when the salt changes can result in
// underbuild.
var extraFingerprintSalts = new ExtraFingerprintSalts(
config,
PipFingerprintingVersion.TwoPhaseV2,
config.Cache.CacheSalt ?? string.Empty,
new Scheduler.DirectoryMembershipFingerprinterRuleSet(config, pathTable.StringTable).ComputeSearchPathToolsHash());
AddFingerprint(hasher, "ExtraFingerprintSalts", extraFingerprintSalts.CalculatedSaltsFingerprint);
}
// Config files
// Caution: Including the content hash of the config file is how changes to the default pip filter
// invalidate a cached graph. If the config file content hash is removed, the values that get
// evaluated because of it must be reflected in the values passed in to this method.
using (var configHasher = new CoreHashingHelper(recordFingerprintString: false))
{
var configFiles = new List <AbsolutePath> {
startUpConfig.ConfigFile
};
try
{
foreach (var configPath in configFiles
.Select(path => path.ToString(pathTable))
.OrderBy(c => c, StringComparer.OrdinalIgnoreCase))
{
AddContentHash(
configHasher,
configPath.ToUpperInvariant(),
fileContentTable.GetAndRecordContentHashAsync(configPath)
.GetAwaiter()
.GetResult()
.VersionedFileIdentityAndContentInfo.FileContentInfo.Hash);
}
fingerprint.ConfigFileHash = configHasher.GenerateHash();
AddFingerprint(hasher, "ConfigFiles", fingerprint.ConfigFileHash);
}
catch (BuildXLException ex)
{
return(LogAndReturnFailure(ex));
}
}
if (!string.IsNullOrEmpty(commitId))
{
using (var commitHasher = new CoreHashingHelper(recordFingerprintString: false))
{
commitHasher.Add("Commit", commitId);
fingerprint.BuildEngineHash = commitHasher.GenerateHash();
}
AddFingerprint(hasher, "BuildEngine", fingerprint.BuildEngineHash);
}
else
{
// BuildXL assemblies. This will invalidate the cached graph if build files change
// or if the serialization format changes.
try
{
Action <string, ContentHash> handleBuildFileAndHash = (buildFile, buildFileHash) =>
{
// Directly add to fingerprint elements for logging, but the hash is represented in the build engine hash
fingerprintTextElements.Add((buildFile, buildFileHash.ToString()));
};
var deployment = testHooks?.AppDeployment ?? AppDeployment.ReadDeploymentManifestFromRunningApp();
fingerprint.BuildEngineHash = deployment.ComputeContentHashBasedFingerprint(fileContentTable, handleBuildFileAndHash);
AddFingerprint(hasher, "BuildEngine", fingerprint.BuildEngineHash);
}
catch (BuildXLException ex)
{
Tracing.Logger.Log.FailedToComputeHashFromDeploymentManifest(loggingContext);
Tracing.Logger.Log.FailedToComputeHashFromDeploymentManifestReason(loggingContext, ex.Message);
return(default(Optional <CompositeGraphFingerprint>));
}
}
AddText(hasher, "HostOS", startUpConfig.CurrentHost.CurrentOS.ToString());
AddText(hasher, "HostCpuArchitecture", startUpConfig.CurrentHost.CpuArchitecture.ToString());
AddText(hasher, "HostIsElevated", CurrentProcess.IsElevated.ToString());
var salt = string.Empty;
if (testHooks?.GraphFingerprintSalt != null)
{
salt += testHooks.GraphFingerprintSalt.Value.ToString();
}
salt += EngineEnvironmentSettings.DebugGraphFingerprintSalt;
if (!string.IsNullOrEmpty(salt))
{
hasher.Add("GraphFingerprintSalt", salt);
}
fingerprint.OverallFingerprint = new ContentFingerprint(hasher.GenerateHash());
Tracing.Logger.Log.ElementsOfConfigurationFingerprint(
loggingContext,
fingerprint.OverallFingerprint.ToString(),
string.Join(Environment.NewLine, fingerprintTextElements.Select(kvp => "\t" + kvp.Item1 + " : " + kvp.Item2)));
return(new Optional <CompositeGraphFingerprint>(fingerprint));
}
// Local functions
void AddInt(CoreHashingHelper hasher, string key, int value)
{
hasher.Add(key, value);
fingerprintTextElements.Add(
(key, value.ToString(CultureInfo.InvariantCulture)));
}
void AddText(CoreHashingHelper hasher, string key, string value)
{
hasher.Add(key, value);
fingerprintTextElements.Add((key, value));
}
void AddFingerprint(CoreHashingHelper hasher, string key, Fingerprint value)
{
hasher.Add(key, value);
fingerprintTextElements.Add((key, value.ToString()));
}
void AddContentHash(CoreHashingHelper hasher, string key, ContentHash value)
{
hasher.Add(key, value);
fingerprintTextElements.Add((key, value.ToString()));
}
}