/// <nodoc />
public LayoutConfiguration(ILayoutConfiguration template, PathRemapper pathRemapper)
{
Contract.Assume(template != null);
Contract.Assume(pathRemapper != null);
PrimaryConfigFile = pathRemapper.Remap(template.PrimaryConfigFile);
SourceDirectory = pathRemapper.Remap(template.SourceDirectory);
OutputDirectory = pathRemapper.Remap(template.OutputDirectory);
ObjectDirectory = pathRemapper.Remap(template.ObjectDirectory);
RedirectedDirectory = pathRemapper.Remap(template.RedirectedDirectory);
FrontEndDirectory = pathRemapper.Remap(template.FrontEndDirectory);
CacheDirectory = pathRemapper.Remap(template.CacheDirectory);
EngineCacheDirectory = pathRemapper.Remap(template.EngineCacheDirectory);
TempDirectory = pathRemapper.Remap(template.TempDirectory);
BuildEngineDirectory = pathRemapper.Remap(template.BuildEngineDirectory);
FileContentTableFile = pathRemapper.Remap(template.FileContentTableFile);
SymlinkDefinitionFile = pathRemapper.Remap(template.SymlinkDefinitionFile);
SchedulerFileChangeTrackerFile = pathRemapper.Remap(template.SchedulerFileChangeTrackerFile);
IncrementalSchedulingStateFile = pathRemapper.Remap(template.IncrementalSchedulingStateFile);
FingerprintStoreDirectory = pathRemapper.Remap(template.FingerprintStoreDirectory);
SharedOpaqueSidebandDirectory = pathRemapper.Remap(template.SharedOpaqueSidebandDirectory);
EmitSpotlightIndexingWarning = template.EmitSpotlightIndexingWarning;
RedirectedUserProfileJunctionRoot = pathRemapper.Remap(template.RedirectedUserProfileJunctionRoot);
ExternalSandboxedProcessDirectory = pathRemapper.Remap(template.ExternalSandboxedProcessDirectory);
}
/// <summary>
/// Tries to fetch symlink file from the cache.
/// </summary>
public static async Task <bool> TryFetchWorkerSymlinkFileAsync(
LoggingContext loggingContext,
PathTable pathTable,
EngineCache cache,
ILayoutConfiguration layoutConfiguration,
WorkerService workerService,
AsyncOut <AbsolutePath> symlinkPathAsyncOut)
{
Contract.Requires(loggingContext != null);
Contract.Requires(pathTable != null);
Contract.Requires(cache != null);
Contract.Requires(workerService != null);
Contract.Requires(symlinkPathAsyncOut != null);
symlinkPathAsyncOut.Value = AbsolutePath.Invalid;
var symlinkFileContentHash = workerService.BuildStartData.SymlinkFileContentHash.ToContentHash();
if (symlinkFileContentHash == WellKnownContentHashes.AbsentFile)
{
// Absent file meaning there is no symlink file to use
return(true);
}
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Attempting to retrieve symlink file with hash '{symlinkFileContentHash}'."));
var maybeLoaded = await cache.ArtifactContentCache.TryLoadAvailableContentAsync(new[] { symlinkFileContentHash });
if (!maybeLoaded.Succeeded)
{
Tracing.Logger.Log.FailedLoadSymlinkFileFromCache(loggingContext, maybeLoaded.Failure.DescribeIncludingInnerFailures());
return(false);
}
var destinationPath = layoutConfiguration.EngineCacheDirectory.Combine(pathTable, SymlinkFileName).Expand(pathTable);
var materializedFile =
await
cache.ArtifactContentCache.TryMaterializeAsync(
FileRealizationMode.HardLinkOrCopy,
destinationPath,
symlinkFileContentHash);
if (!materializedFile.Succeeded)
{
Tracing.Logger.Log.FailedMaterializeSymlinkFileFromCache(
loggingContext,
destinationPath.ExpandedPath,
materializedFile.Failure.DescribeIncludingInnerFailures());
return(false);
}
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Symlink file with hash '{symlinkFileContentHash}' materialized to location '{destinationPath}'."));
symlinkPathAsyncOut.Value = destinationPath.Path;
return(true);
}
/// <nodoc />
public LayoutConfiguration(ILayoutConfiguration template, PathRemapper pathRemapper)
{
Contract.Assume(template != null);
Contract.Assume(pathRemapper != null);
PrimaryConfigFile = pathRemapper.Remap(template.PrimaryConfigFile);
SourceDirectory = pathRemapper.Remap(template.SourceDirectory);
OutputDirectory = pathRemapper.Remap(template.OutputDirectory);
ObjectDirectory = pathRemapper.Remap(template.ObjectDirectory);
FrontEndDirectory = pathRemapper.Remap(template.FrontEndDirectory);
CacheDirectory = pathRemapper.Remap(template.CacheDirectory);
EngineCacheDirectory = pathRemapper.Remap(template.EngineCacheDirectory);
TempDirectory = pathRemapper.Remap(template.TempDirectory);
BuildXlBinDirectory = pathRemapper.Remap(template.BuildXlBinDirectory);
FileContentTableFile = pathRemapper.Remap(template.FileContentTableFile);
SymlinkDefinitionFile = pathRemapper.Remap(template.SymlinkDefinitionFile);
SchedulerFileChangeTrackerFile = pathRemapper.Remap(template.SchedulerFileChangeTrackerFile);
IncrementalSchedulingStateFile = pathRemapper.Remap(template.IncrementalSchedulingStateFile);
FingerprintStoreDirectory = pathRemapper.Remap(template.FingerprintStoreDirectory);
}
public static MountsTable CreateAndRegister(
LoggingContext loggingContext,
BuildXLContext context,
IConfiguration configuration,
[CanBeNull] IReadOnlyDictionary <string, string> properties)
{
Contract.Requires(context != null);
Contract.Requires(configuration != null);
Contract.Requires(loggingContext != null);
ILayoutConfiguration layout = configuration.Layout;
IReadOnlyList <IResolverSettings> resolverSettings = configuration.Resolvers;
var table = new MountsTable(loggingContext, context);
// If any resolver settings allows for a writable source directory, then the source directory is writable
var writableSourceRoot = resolverSettings.Any(resolverSetting => resolverSetting.AllowWritableSourceDirectory);
table.AddStaticMount("BuildEnginePath", layout.BuildEngineDirectory, isWriteable: false);
table.AddStaticMount("SourceRoot", layout.SourceDirectory, isWriteable: writableSourceRoot);
table.AddStaticMount("ObjectRoot", layout.ObjectDirectory, isWriteable: true, isScrubbable: true);
table.AddStaticMount(
"LogsDirectory",
configuration.Logging.RedirectedLogsDirectory.IsValid
? configuration.Logging.RedirectedLogsDirectory
: configuration.Logging.LogsDirectory,
isWriteable: true,
allowCreateDirectory: true);
if (layout.FrontEndDirectory.IsValid)
{
// This location is used only for storing nuget packages and generated specs so far.
table.AddStaticMount("FrontEnd", layout.FrontEndDirectory, isWriteable: true, isScrubbable: false);
}
if (layout.TempDirectory.IsValid)
{
table.AddStaticMount("TempRoot", layout.TempDirectory, isWriteable: true, isScrubbable: true);
}
// Cross Plat supported MountPoints
table.AddStaticSystemMount("ProgramData", Environment.SpecialFolder.CommonApplicationData);
table.AddStaticSystemMount("ProgramFiles", Environment.SpecialFolder.ProgramFiles, trackSourceFileChanges: true);
table.AddStaticSystemMount("System", Environment.SpecialFolder.System);
if (!layout.RedirectedUserProfileJunctionRoot.IsValid)
{
table.AddStaticSystemMount("UserProfile", Environment.SpecialFolder.UserProfile);
table.AddStaticSystemMount("AppData", Environment.SpecialFolder.ApplicationData, allowCreateDirectory: true);
table.AddStaticSystemMount("LocalAppData", Environment.SpecialFolder.LocalApplicationData, allowCreateDirectory: true);
}
else
{
// User profile is redirected; need to use the paths specified in the env block.
Contract.Assert(properties != null);
RegisterRedirectedMount(context, properties, table, "UserProfile");
RegisterRedirectedMount(context, properties, table, "AppData", allowCreateDirectory: true);
RegisterRedirectedMount(context, properties, table, "LocalAppData", allowCreateDirectory: true);
}
if (!OperatingSystemHelper.IsUnixOS)
{
// Add system mounts that are Windows Only
table.AddStaticSystemMount("Windows", Environment.SpecialFolder.Windows);
table.AddStaticSystemMount("ProgramFilesX86", Environment.SpecialFolder.ProgramFilesX86, trackSourceFileChanges: true);
table.AddStaticSystemMount("CommonProgramFiles", Environment.SpecialFolder.CommonProgramFiles, trackSourceFileChanges: true);
table.AddStaticSystemMount("CommonProgramFilesX86", Environment.SpecialFolder.CommonProgramFilesX86, trackSourceFileChanges: true);
if (!layout.RedirectedUserProfileJunctionRoot.IsValid)
{
table.AddStaticSystemMount("InternetCache", Environment.SpecialFolder.InternetCache);
table.AddStaticSystemMount("InternetHistory", Environment.SpecialFolder.History);
table.AddStaticSystemMount("INetCookies", Environment.SpecialFolder.Cookies, allowCreateDirectory: true);
table.AddStaticSystemMount("LocalLow", FileUtilities.KnownFolderLocalLow);
}
else
{
// User profile is redirected; need to use the paths specified in the env block.
Contract.Assert(properties != null);
RegisterRedirectedMount(context, properties, table, "InternetCache");
RegisterRedirectedMount(context, properties, table, "InternetHistory");
RegisterRedirectedMount(context, properties, table, "INetCookies", allowCreateDirectory: true);
RegisterRedirectedMount(context, properties, table, "LocalLow");
}
}
else
{
table.AddStaticSystemMount("Applications", MacPaths.Applications, trackSourceFileChanges: true);
table.AddStaticSystemMount("Bin", MacPaths.Bin, trackSourceFileChanges: true);
table.AddStaticSystemMount("UsrBin", MacPaths.UsrBin, trackSourceFileChanges: true);
table.AddStaticSystemMount("UsrInclude", MacPaths.UsrInclude, trackSourceFileChanges: true);
table.AddStaticSystemMount("UsrLib", MacPaths.UsrLib, trackSourceFileChanges: true);
table.AddStaticSystemMount("Library", MacPaths.Library, trackSourceFileChanges: true);
table.AddStaticSystemMount("UserProvisioning", MacPaths.UserProvisioning, trackSourceFileChanges: true);
}
return(table);
}
private static Optional <CompositeGraphFingerprint> GenerateHash(
LoggingContext loggingContext,
IStartupConfiguration startUpConfig,
IConfiguration config,
PathTable pathTable,
IEvaluationFilter partialEvaluationData,
FileContentTable fileContentTable,
string commitId,
EngineTestHooksData testHooks)
{
ILayoutConfiguration layout = config.Layout;
ILoggingConfiguration logging = config.Logging;
var fingerprintTextElements = new List <(string, string)>();
using (var hasher = new CoreHashingHelper(recordFingerprintString: true))
{
CompositeGraphFingerprint fingerprint = CompositeGraphFingerprint.Zero;
AddInt(hasher, "version", GraphFingerprintVersion);
using (var qualifierHasher = new CoreHashingHelper(recordFingerprintString: false))
{
foreach (string qualifier in startUpConfig.QualifierIdentifiers.OrderBy(q => q))
{
AddText(qualifierHasher, "qualifier", qualifier);
}
fingerprint.QualifierHash = qualifierHasher.GenerateHash();
AddFingerprint(hasher, "Qualifiers", fingerprint.QualifierHash);
}
if (partialEvaluationData != null && partialEvaluationData.CanPerformPartialEvaluation)
{
using (var evaluationFilterHasher = new CoreHashingHelper(recordFingerprintString: false))
{
foreach (string value in partialEvaluationData.ValueNamesToResolveAsStrings.OrderBy(v => v))
{
AddText(evaluationFilterHasher, "valueName", value);
}
foreach (string value in partialEvaluationData.ValueDefinitionRootsToResolveAsStrings.OrderBy(v => v))
{
AddText(evaluationFilterHasher, "valuePath", value);
}
foreach (string value in partialEvaluationData.ModulesToResolveAsStrings.OrderBy(v => v))
{
AddText(evaluationFilterHasher, "moduleName", value);
}
fingerprint.FilterHash = evaluationFilterHasher.GenerateHash();
AddFingerprint(hasher, "Values", fingerprint.FilterHash);
}
}
// These paths get embedded in the result of evaluation. So if any change we must re-evaluate
AddText(hasher, "ObjectDirectoryPath", layout.ObjectDirectory.IsValid ? layout.ObjectDirectory.ToString(pathTable) : "::null::");
AddText(hasher, "TempDirectoryPath", layout.TempDirectory.IsValid ? layout.TempDirectory.ToString(pathTable) : "::null::");
AddText(hasher, "SourceDirectoryPath", layout.SourceDirectory.IsValid ? layout.SourceDirectory.ToString(pathTable) : "::null::");
// All paths in the graph are relative to 'substTarget' (hence, 'substTarget' must be a part of the fingerprint, but 'substSource' need not be).
AddText(hasher, "substTarget", logging.SubstTarget.IsValid ? logging.SubstTarget.ToString(pathTable) : "::null::");
AddText(hasher, "IsCompressed", config.Engine.CompressGraphFiles ? "true" : "false");
AddText(hasher, "IsSkipHashSourceFile", config.Schedule.SkipHashSourceFile ? "true" : "false");
// Pip static fingerprints are not always computed because computing them slows down the graph construction by 10%-13%.
// Thus, the pip graph may and may not contain pip static fingerprints. To avoid unexpected result due to graph cache hit,
// we temporarily add the option for computing pip static fingerprints as part of our graph fingerprint until the fingerprints
// are always computed; see Task 1291638.
AddText(hasher, "ComputePipStaticFingerprints", config.Schedule.ComputePipStaticFingerprints.ToString());
if (config.Schedule.ComputePipStaticFingerprints)
{
// Pip static fingerprints are part of the graph and include the extra fingerprint salts.
// Thus, when pip static fingerprints are computed, any change to the salt will invalidate the graph because
// the pip static fingerprints will no longer be valid. Reusing the graph when the salt changes can result in
// underbuild.
var extraFingerprintSalts = new ExtraFingerprintSalts(
config,
PipFingerprintingVersion.TwoPhaseV2,
config.Cache.CacheSalt ?? string.Empty,
new Scheduler.DirectoryMembershipFingerprinterRuleSet(config, pathTable.StringTable).ComputeSearchPathToolsHash());
AddFingerprint(hasher, "ExtraFingerprintSalts", extraFingerprintSalts.CalculatedSaltsFingerprint);
}
// Config files
// Caution: Including the content hash of the config file is how changes to the default pip filter
// invalidate a cached graph. If the config file content hash is removed, the values that get
// evaluated because of it must be reflected in the values passed in to this method.
using (var configHasher = new CoreHashingHelper(recordFingerprintString: false))
{
var configFiles = new List <AbsolutePath> {
startUpConfig.ConfigFile
};
try
{
foreach (var configPath in configFiles
.Select(path => path.ToString(pathTable))
.OrderBy(c => c, StringComparer.OrdinalIgnoreCase))
{
AddContentHash(
configHasher,
configPath.ToUpperInvariant(),
fileContentTable.GetAndRecordContentHashAsync(configPath)
.GetAwaiter()
.GetResult()
.VersionedFileIdentityAndContentInfo.FileContentInfo.Hash);
}
fingerprint.ConfigFileHash = configHasher.GenerateHash();
AddFingerprint(hasher, "ConfigFiles", fingerprint.ConfigFileHash);
}
catch (BuildXLException ex)
{
return(LogAndReturnFailure(ex));
}
}
if (!string.IsNullOrEmpty(commitId))
{
using (var commitHasher = new CoreHashingHelper(recordFingerprintString: false))
{
commitHasher.Add("Commit", commitId);
fingerprint.BuildEngineHash = commitHasher.GenerateHash();
}
AddFingerprint(hasher, "BuildEngine", fingerprint.BuildEngineHash);
}
else
{
// BuildXL assemblies. This will invalidate the cached graph if build files change
// or if the serialization format changes.
try
{
Action <string, ContentHash> handleBuildFileAndHash = (buildFile, buildFileHash) =>
{
// Directly add to fingerprint elements for logging, but the hash is represented in the build engine hash
fingerprintTextElements.Add((buildFile, buildFileHash.ToString()));
};
var deployment = testHooks?.AppDeployment ?? AppDeployment.ReadDeploymentManifestFromRunningApp();
fingerprint.BuildEngineHash = deployment.ComputeContentHashBasedFingerprint(fileContentTable, handleBuildFileAndHash);
AddFingerprint(hasher, "BuildEngine", fingerprint.BuildEngineHash);
}
catch (BuildXLException ex)
{
Tracing.Logger.Log.FailedToComputeHashFromDeploymentManifest(loggingContext);
Tracing.Logger.Log.FailedToComputeHashFromDeploymentManifestReason(loggingContext, ex.Message);
return(default(Optional <CompositeGraphFingerprint>));
}
}
AddText(hasher, "HostOS", startUpConfig.CurrentHost.CurrentOS.ToString());
AddText(hasher, "HostCpuArchitecture", startUpConfig.CurrentHost.CpuArchitecture.ToString());
AddText(hasher, "HostIsElevated", CurrentProcess.IsElevated.ToString());
var salt = string.Empty;
if (testHooks?.GraphFingerprintSalt != null)
{
salt += testHooks.GraphFingerprintSalt.Value.ToString();
}
salt += EngineEnvironmentSettings.DebugGraphFingerprintSalt;
if (!string.IsNullOrEmpty(salt))
{
hasher.Add("GraphFingerprintSalt", salt);
}
fingerprint.OverallFingerprint = new ContentFingerprint(hasher.GenerateHash());
Tracing.Logger.Log.ElementsOfConfigurationFingerprint(
loggingContext,
fingerprint.OverallFingerprint.ToString(),
string.Join(Environment.NewLine, fingerprintTextElements.Select(kvp => "\t" + kvp.Item1 + " : " + kvp.Item2)));
return(new Optional <CompositeGraphFingerprint>(fingerprint));
}
// Local functions
void AddInt(CoreHashingHelper hasher, string key, int value)
{
hasher.Add(key, value);
fingerprintTextElements.Add(
(key, value.ToString(CultureInfo.InvariantCulture)));
}
void AddText(CoreHashingHelper hasher, string key, string value)
{
hasher.Add(key, value);
fingerprintTextElements.Add((key, value));
}
void AddFingerprint(CoreHashingHelper hasher, string key, Fingerprint value)
{
hasher.Add(key, value);
fingerprintTextElements.Add((key, value.ToString()));
}
void AddContentHash(CoreHashingHelper hasher, string key, ContentHash value)
{
hasher.Add(key, value);
fingerprintTextElements.Add((key, value.ToString()));
}
}
protected TextLayout(ILayoutConfiguration configuration)
{
}
public XmlFormatter(ILayoutConfiguration layout_config)
{
this.layoutconfiguration = (IXmlLayoutConfiguration)layout_config;
CreateXml();
}
//Functions
protected LayoutBase()
{
v_configuration = new LayoutConfiguration();
}
protected XmlLayout(ILayoutConfiguration configuration)
{
v_configuration = (IXmlLayoutConfiguration)configuration;
base.formatter = new XmlFormatter(v_configuration);
CreateStartXml();
}
