/// <nodoc /> public LayoutConfiguration(ILayoutConfiguration template, PathRemapper pathRemapper) { Contract.Assume(template != null); Contract.Assume(pathRemapper != null); PrimaryConfigFile = pathRemapper.Remap(template.PrimaryConfigFile); SourceDirectory = pathRemapper.Remap(template.SourceDirectory); OutputDirectory = pathRemapper.Remap(template.OutputDirectory); ObjectDirectory = pathRemapper.Remap(template.ObjectDirectory); RedirectedDirectory = pathRemapper.Remap(template.RedirectedDirectory); FrontEndDirectory = pathRemapper.Remap(template.FrontEndDirectory); CacheDirectory = pathRemapper.Remap(template.CacheDirectory); EngineCacheDirectory = pathRemapper.Remap(template.EngineCacheDirectory); TempDirectory = pathRemapper.Remap(template.TempDirectory); BuildEngineDirectory = pathRemapper.Remap(template.BuildEngineDirectory); FileContentTableFile = pathRemapper.Remap(template.FileContentTableFile); SymlinkDefinitionFile = pathRemapper.Remap(template.SymlinkDefinitionFile); SchedulerFileChangeTrackerFile = pathRemapper.Remap(template.SchedulerFileChangeTrackerFile); IncrementalSchedulingStateFile = pathRemapper.Remap(template.IncrementalSchedulingStateFile); FingerprintStoreDirectory = pathRemapper.Remap(template.FingerprintStoreDirectory); SharedOpaqueSidebandDirectory = pathRemapper.Remap(template.SharedOpaqueSidebandDirectory); EmitSpotlightIndexingWarning = template.EmitSpotlightIndexingWarning; RedirectedUserProfileJunctionRoot = pathRemapper.Remap(template.RedirectedUserProfileJunctionRoot); ExternalSandboxedProcessDirectory = pathRemapper.Remap(template.ExternalSandboxedProcessDirectory); }
/// <summary> /// Tries to fetch symlink file from the cache. /// </summary> public static async Task <bool> TryFetchWorkerSymlinkFileAsync( LoggingContext loggingContext, PathTable pathTable, EngineCache cache, ILayoutConfiguration layoutConfiguration, WorkerService workerService, AsyncOut <AbsolutePath> symlinkPathAsyncOut) { Contract.Requires(loggingContext != null); Contract.Requires(pathTable != null); Contract.Requires(cache != null); Contract.Requires(workerService != null); Contract.Requires(symlinkPathAsyncOut != null); symlinkPathAsyncOut.Value = AbsolutePath.Invalid; var symlinkFileContentHash = workerService.BuildStartData.SymlinkFileContentHash.ToContentHash(); if (symlinkFileContentHash == WellKnownContentHashes.AbsentFile) { // Absent file meaning there is no symlink file to use return(true); } Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Attempting to retrieve symlink file with hash '{symlinkFileContentHash}'.")); var maybeLoaded = await cache.ArtifactContentCache.TryLoadAvailableContentAsync(new[] { symlinkFileContentHash }); if (!maybeLoaded.Succeeded) { Tracing.Logger.Log.FailedLoadSymlinkFileFromCache(loggingContext, maybeLoaded.Failure.DescribeIncludingInnerFailures()); return(false); } var destinationPath = layoutConfiguration.EngineCacheDirectory.Combine(pathTable, SymlinkFileName).Expand(pathTable); var materializedFile = await cache.ArtifactContentCache.TryMaterializeAsync( FileRealizationMode.HardLinkOrCopy, destinationPath, symlinkFileContentHash); if (!materializedFile.Succeeded) { Tracing.Logger.Log.FailedMaterializeSymlinkFileFromCache( loggingContext, destinationPath.ExpandedPath, materializedFile.Failure.DescribeIncludingInnerFailures()); return(false); } Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Symlink file with hash '{symlinkFileContentHash}' materialized to location '{destinationPath}'.")); symlinkPathAsyncOut.Value = destinationPath.Path; return(true); }
/// <nodoc /> public LayoutConfiguration(ILayoutConfiguration template, PathRemapper pathRemapper) { Contract.Assume(template != null); Contract.Assume(pathRemapper != null); PrimaryConfigFile = pathRemapper.Remap(template.PrimaryConfigFile); SourceDirectory = pathRemapper.Remap(template.SourceDirectory); OutputDirectory = pathRemapper.Remap(template.OutputDirectory); ObjectDirectory = pathRemapper.Remap(template.ObjectDirectory); FrontEndDirectory = pathRemapper.Remap(template.FrontEndDirectory); CacheDirectory = pathRemapper.Remap(template.CacheDirectory); EngineCacheDirectory = pathRemapper.Remap(template.EngineCacheDirectory); TempDirectory = pathRemapper.Remap(template.TempDirectory); BuildXlBinDirectory = pathRemapper.Remap(template.BuildXlBinDirectory); FileContentTableFile = pathRemapper.Remap(template.FileContentTableFile); SymlinkDefinitionFile = pathRemapper.Remap(template.SymlinkDefinitionFile); SchedulerFileChangeTrackerFile = pathRemapper.Remap(template.SchedulerFileChangeTrackerFile); IncrementalSchedulingStateFile = pathRemapper.Remap(template.IncrementalSchedulingStateFile); FingerprintStoreDirectory = pathRemapper.Remap(template.FingerprintStoreDirectory); }
public static MountsTable CreateAndRegister( LoggingContext loggingContext, BuildXLContext context, IConfiguration configuration, [CanBeNull] IReadOnlyDictionary <string, string> properties) { Contract.Requires(context != null); Contract.Requires(configuration != null); Contract.Requires(loggingContext != null); ILayoutConfiguration layout = configuration.Layout; IReadOnlyList <IResolverSettings> resolverSettings = configuration.Resolvers; var table = new MountsTable(loggingContext, context); // If any resolver settings allows for a writable source directory, then the source directory is writable var writableSourceRoot = resolverSettings.Any(resolverSetting => resolverSetting.AllowWritableSourceDirectory); table.AddStaticMount("BuildEnginePath", layout.BuildEngineDirectory, isWriteable: false); table.AddStaticMount("SourceRoot", layout.SourceDirectory, isWriteable: writableSourceRoot); table.AddStaticMount("ObjectRoot", layout.ObjectDirectory, isWriteable: true, isScrubbable: true); table.AddStaticMount( "LogsDirectory", configuration.Logging.RedirectedLogsDirectory.IsValid ? configuration.Logging.RedirectedLogsDirectory : configuration.Logging.LogsDirectory, isWriteable: true, allowCreateDirectory: true); if (layout.FrontEndDirectory.IsValid) { // This location is used only for storing nuget packages and generated specs so far. table.AddStaticMount("FrontEnd", layout.FrontEndDirectory, isWriteable: true, isScrubbable: false); } if (layout.TempDirectory.IsValid) { table.AddStaticMount("TempRoot", layout.TempDirectory, isWriteable: true, isScrubbable: true); } // Cross Plat supported MountPoints table.AddStaticSystemMount("ProgramData", Environment.SpecialFolder.CommonApplicationData); table.AddStaticSystemMount("ProgramFiles", Environment.SpecialFolder.ProgramFiles, trackSourceFileChanges: true); table.AddStaticSystemMount("System", Environment.SpecialFolder.System); if (!layout.RedirectedUserProfileJunctionRoot.IsValid) { table.AddStaticSystemMount("UserProfile", Environment.SpecialFolder.UserProfile); table.AddStaticSystemMount("AppData", Environment.SpecialFolder.ApplicationData, allowCreateDirectory: true); table.AddStaticSystemMount("LocalAppData", Environment.SpecialFolder.LocalApplicationData, allowCreateDirectory: true); } else { // User profile is redirected; need to use the paths specified in the env block. Contract.Assert(properties != null); RegisterRedirectedMount(context, properties, table, "UserProfile"); RegisterRedirectedMount(context, properties, table, "AppData", allowCreateDirectory: true); RegisterRedirectedMount(context, properties, table, "LocalAppData", allowCreateDirectory: true); } if (!OperatingSystemHelper.IsUnixOS) { // Add system mounts that are Windows Only table.AddStaticSystemMount("Windows", Environment.SpecialFolder.Windows); table.AddStaticSystemMount("ProgramFilesX86", Environment.SpecialFolder.ProgramFilesX86, trackSourceFileChanges: true); table.AddStaticSystemMount("CommonProgramFiles", Environment.SpecialFolder.CommonProgramFiles, trackSourceFileChanges: true); table.AddStaticSystemMount("CommonProgramFilesX86", Environment.SpecialFolder.CommonProgramFilesX86, trackSourceFileChanges: true); if (!layout.RedirectedUserProfileJunctionRoot.IsValid) { table.AddStaticSystemMount("InternetCache", Environment.SpecialFolder.InternetCache); table.AddStaticSystemMount("InternetHistory", Environment.SpecialFolder.History); table.AddStaticSystemMount("INetCookies", Environment.SpecialFolder.Cookies, allowCreateDirectory: true); table.AddStaticSystemMount("LocalLow", FileUtilities.KnownFolderLocalLow); } else { // User profile is redirected; need to use the paths specified in the env block. Contract.Assert(properties != null); RegisterRedirectedMount(context, properties, table, "InternetCache"); RegisterRedirectedMount(context, properties, table, "InternetHistory"); RegisterRedirectedMount(context, properties, table, "INetCookies", allowCreateDirectory: true); RegisterRedirectedMount(context, properties, table, "LocalLow"); } } else { table.AddStaticSystemMount("Applications", MacPaths.Applications, trackSourceFileChanges: true); table.AddStaticSystemMount("Bin", MacPaths.Bin, trackSourceFileChanges: true); table.AddStaticSystemMount("UsrBin", MacPaths.UsrBin, trackSourceFileChanges: true); table.AddStaticSystemMount("UsrInclude", MacPaths.UsrInclude, trackSourceFileChanges: true); table.AddStaticSystemMount("UsrLib", MacPaths.UsrLib, trackSourceFileChanges: true); table.AddStaticSystemMount("Library", MacPaths.Library, trackSourceFileChanges: true); table.AddStaticSystemMount("UserProvisioning", MacPaths.UserProvisioning, trackSourceFileChanges: true); } return(table); }
private static Optional <CompositeGraphFingerprint> GenerateHash( LoggingContext loggingContext, IStartupConfiguration startUpConfig, IConfiguration config, PathTable pathTable, IEvaluationFilter partialEvaluationData, FileContentTable fileContentTable, string commitId, EngineTestHooksData testHooks) { ILayoutConfiguration layout = config.Layout; ILoggingConfiguration logging = config.Logging; var fingerprintTextElements = new List <(string, string)>(); using (var hasher = new CoreHashingHelper(recordFingerprintString: true)) { CompositeGraphFingerprint fingerprint = CompositeGraphFingerprint.Zero; AddInt(hasher, "version", GraphFingerprintVersion); using (var qualifierHasher = new CoreHashingHelper(recordFingerprintString: false)) { foreach (string qualifier in startUpConfig.QualifierIdentifiers.OrderBy(q => q)) { AddText(qualifierHasher, "qualifier", qualifier); } fingerprint.QualifierHash = qualifierHasher.GenerateHash(); AddFingerprint(hasher, "Qualifiers", fingerprint.QualifierHash); } if (partialEvaluationData != null && partialEvaluationData.CanPerformPartialEvaluation) { using (var evaluationFilterHasher = new CoreHashingHelper(recordFingerprintString: false)) { foreach (string value in partialEvaluationData.ValueNamesToResolveAsStrings.OrderBy(v => v)) { AddText(evaluationFilterHasher, "valueName", value); } foreach (string value in partialEvaluationData.ValueDefinitionRootsToResolveAsStrings.OrderBy(v => v)) { AddText(evaluationFilterHasher, "valuePath", value); } foreach (string value in partialEvaluationData.ModulesToResolveAsStrings.OrderBy(v => v)) { AddText(evaluationFilterHasher, "moduleName", value); } fingerprint.FilterHash = evaluationFilterHasher.GenerateHash(); AddFingerprint(hasher, "Values", fingerprint.FilterHash); } } // These paths get embedded in the result of evaluation. So if any change we must re-evaluate AddText(hasher, "ObjectDirectoryPath", layout.ObjectDirectory.IsValid ? layout.ObjectDirectory.ToString(pathTable) : "::null::"); AddText(hasher, "TempDirectoryPath", layout.TempDirectory.IsValid ? layout.TempDirectory.ToString(pathTable) : "::null::"); AddText(hasher, "SourceDirectoryPath", layout.SourceDirectory.IsValid ? layout.SourceDirectory.ToString(pathTable) : "::null::"); // All paths in the graph are relative to 'substTarget' (hence, 'substTarget' must be a part of the fingerprint, but 'substSource' need not be). AddText(hasher, "substTarget", logging.SubstTarget.IsValid ? logging.SubstTarget.ToString(pathTable) : "::null::"); AddText(hasher, "IsCompressed", config.Engine.CompressGraphFiles ? "true" : "false"); AddText(hasher, "IsSkipHashSourceFile", config.Schedule.SkipHashSourceFile ? "true" : "false"); // Pip static fingerprints are not always computed because computing them slows down the graph construction by 10%-13%. // Thus, the pip graph may and may not contain pip static fingerprints. To avoid unexpected result due to graph cache hit, // we temporarily add the option for computing pip static fingerprints as part of our graph fingerprint until the fingerprints // are always computed; see Task 1291638. AddText(hasher, "ComputePipStaticFingerprints", config.Schedule.ComputePipStaticFingerprints.ToString()); if (config.Schedule.ComputePipStaticFingerprints) { // Pip static fingerprints are part of the graph and include the extra fingerprint salts. // Thus, when pip static fingerprints are computed, any change to the salt will invalidate the graph because // the pip static fingerprints will no longer be valid. Reusing the graph when the salt changes can result in // underbuild. var extraFingerprintSalts = new ExtraFingerprintSalts( config, PipFingerprintingVersion.TwoPhaseV2, config.Cache.CacheSalt ?? string.Empty, new Scheduler.DirectoryMembershipFingerprinterRuleSet(config, pathTable.StringTable).ComputeSearchPathToolsHash()); AddFingerprint(hasher, "ExtraFingerprintSalts", extraFingerprintSalts.CalculatedSaltsFingerprint); } // Config files // Caution: Including the content hash of the config file is how changes to the default pip filter // invalidate a cached graph. If the config file content hash is removed, the values that get // evaluated because of it must be reflected in the values passed in to this method. using (var configHasher = new CoreHashingHelper(recordFingerprintString: false)) { var configFiles = new List <AbsolutePath> { startUpConfig.ConfigFile }; try { foreach (var configPath in configFiles .Select(path => path.ToString(pathTable)) .OrderBy(c => c, StringComparer.OrdinalIgnoreCase)) { AddContentHash( configHasher, configPath.ToUpperInvariant(), fileContentTable.GetAndRecordContentHashAsync(configPath) .GetAwaiter() .GetResult() .VersionedFileIdentityAndContentInfo.FileContentInfo.Hash); } fingerprint.ConfigFileHash = configHasher.GenerateHash(); AddFingerprint(hasher, "ConfigFiles", fingerprint.ConfigFileHash); } catch (BuildXLException ex) { return(LogAndReturnFailure(ex)); } } if (!string.IsNullOrEmpty(commitId)) { using (var commitHasher = new CoreHashingHelper(recordFingerprintString: false)) { commitHasher.Add("Commit", commitId); fingerprint.BuildEngineHash = commitHasher.GenerateHash(); } AddFingerprint(hasher, "BuildEngine", fingerprint.BuildEngineHash); } else { // BuildXL assemblies. This will invalidate the cached graph if build files change // or if the serialization format changes. try { Action <string, ContentHash> handleBuildFileAndHash = (buildFile, buildFileHash) => { // Directly add to fingerprint elements for logging, but the hash is represented in the build engine hash fingerprintTextElements.Add((buildFile, buildFileHash.ToString())); }; var deployment = testHooks?.AppDeployment ?? AppDeployment.ReadDeploymentManifestFromRunningApp(); fingerprint.BuildEngineHash = deployment.ComputeContentHashBasedFingerprint(fileContentTable, handleBuildFileAndHash); AddFingerprint(hasher, "BuildEngine", fingerprint.BuildEngineHash); } catch (BuildXLException ex) { Tracing.Logger.Log.FailedToComputeHashFromDeploymentManifest(loggingContext); Tracing.Logger.Log.FailedToComputeHashFromDeploymentManifestReason(loggingContext, ex.Message); return(default(Optional <CompositeGraphFingerprint>)); } } AddText(hasher, "HostOS", startUpConfig.CurrentHost.CurrentOS.ToString()); AddText(hasher, "HostCpuArchitecture", startUpConfig.CurrentHost.CpuArchitecture.ToString()); AddText(hasher, "HostIsElevated", CurrentProcess.IsElevated.ToString()); var salt = string.Empty; if (testHooks?.GraphFingerprintSalt != null) { salt += testHooks.GraphFingerprintSalt.Value.ToString(); } salt += EngineEnvironmentSettings.DebugGraphFingerprintSalt; if (!string.IsNullOrEmpty(salt)) { hasher.Add("GraphFingerprintSalt", salt); } fingerprint.OverallFingerprint = new ContentFingerprint(hasher.GenerateHash()); Tracing.Logger.Log.ElementsOfConfigurationFingerprint( loggingContext, fingerprint.OverallFingerprint.ToString(), string.Join(Environment.NewLine, fingerprintTextElements.Select(kvp => "\t" + kvp.Item1 + " : " + kvp.Item2))); return(new Optional <CompositeGraphFingerprint>(fingerprint)); } // Local functions void AddInt(CoreHashingHelper hasher, string key, int value) { hasher.Add(key, value); fingerprintTextElements.Add( (key, value.ToString(CultureInfo.InvariantCulture))); } void AddText(CoreHashingHelper hasher, string key, string value) { hasher.Add(key, value); fingerprintTextElements.Add((key, value)); } void AddFingerprint(CoreHashingHelper hasher, string key, Fingerprint value) { hasher.Add(key, value); fingerprintTextElements.Add((key, value.ToString())); } void AddContentHash(CoreHashingHelper hasher, string key, ContentHash value) { hasher.Add(key, value); fingerprintTextElements.Add((key, value.ToString())); } }
protected TextLayout(ILayoutConfiguration configuration) { }
public XmlFormatter(ILayoutConfiguration layout_config) { this.layoutconfiguration = (IXmlLayoutConfiguration)layout_config; CreateXml(); }
//Functions protected LayoutBase() { v_configuration = new LayoutConfiguration(); }
protected XmlLayout(ILayoutConfiguration configuration) { v_configuration = (IXmlLayoutConfiguration)configuration; base.formatter = new XmlFormatter(v_configuration); CreateStartXml(); }