Esempio n. 1
0
        /// <summary>
        /// Creates an instance of <see cref="PipGraphFragmentBuilder"/>.
        /// </summary>
        public PipGraphFragmentBuilder(
            PipExecutionContext pipExecutionContext,
            IConfiguration configuration,
            PathExpander pathExpander)
        {
            Contract.Requires(pipExecutionContext != null);

            Configuration          = configuration;
            m_pipExecutionContext  = pipExecutionContext;
            m_lazyApiServerMoniker = configuration.Schedule.UseFixedApiServerMoniker
                ? Lazy.Create(() => IpcFactory.GetFixedMoniker())
                : Lazy.Create(() => IpcFactory.GetProvider().CreateNewMoniker());
            SealDirectoryTable = new SealedDirectoryTable(m_pipExecutionContext.PathTable);

            if (configuration.Schedule.ComputePipStaticFingerprints)
            {
                var extraFingerprintSalts = new ExtraFingerprintSalts(
                    configuration,
                    PipFingerprintingVersion.TwoPhaseV2,
                    configuration.Cache.CacheSalt,
                    new DirectoryMembershipFingerprinterRuleSet(configuration, pipExecutionContext.StringTable).ComputeSearchPathToolsHash());

                m_pipStaticFingerprinter = new PipStaticFingerprinter(
                    pipExecutionContext.PathTable,
                    sealDirectoryFingerprintLookup: null,
                    directoryProducerFingerprintLookup: null,
                    extraFingerprintSalts: extraFingerprintSalts,
                    pathExpander: pathExpander)
                {
                    FingerprintTextEnabled = configuration.Schedule.LogPipStaticFingerprintTexts
                };
            }
        }
Esempio n. 2
0
 private PipContentFingerprinter CreateFingerprinter(FileArtifact executable)
 {
     return(new PipContentFingerprinter(
                m_context.PathTable,
                PipFingerprinterTests.GetContentHashLookup(executable),
                ExtraFingerprintSalts.Default())
     {
         FingerprintTextEnabled = true
     });
 }
Esempio n. 3
0
            public PipQueueTestExecutionEnvironment(BuildXLContext context, IConfiguration configuration, PipTable pipTable, string tempDirectory, ISandboxConnection SandboxConnection = null)
            {
                Contract.Requires(context != null);
                Contract.Requires(configuration != null);

                Context              = context;
                LoggingContext       = CreateLoggingContextForTest();
                Configuration        = configuration;
                FileContentTable     = FileContentTable.CreateNew();
                ContentFingerprinter = new PipContentFingerprinter(
                    context.PathTable,
                    artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
                    ExtraFingerprintSalts.Default(),
                    pathExpander: PathExpander);
                PipTable            = pipTable;
                PipFragmentRenderer = this.CreatePipFragmentRenderer();
                IpcProvider         = IpcFactory.GetProvider();
                var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);

                Cache = InMemoryCacheFactory.Create();
                LocalDiskContentStore = new LocalDiskContentStore(LoggingContext, context.PathTable, FileContentTable, tracker);

                m_sandboxConnectionKext  = SandboxConnection;
                m_expectedWrittenContent = new ConcurrentDictionary <FileArtifact, ContentHash>();
                m_wellKnownFiles         = new ConcurrentDictionary <FileArtifact, ContentHash>();
                m_producers      = new ConcurrentDictionary <FileArtifact, Pip>();
                m_filesystemView = new TestPipGraphFilesystemView(Context.PathTable);
                var fileSystemView = new FileSystemView(Context.PathTable, m_filesystemView, LocalDiskContentStore);

                TempCleaner = new TestMoveDeleteCleaner(tempDirectory);

                State = new PipExecutionState(
                    configuration,
                    cache: new PipTwoPhaseCache(LoggingContext, Cache, context, PathExpander),
                    unsafeConfiguration: configuration.Sandbox.UnsafeSandboxConfiguration,
                    preserveOutputsSalt: ContentHashingUtilities.CreateRandom(),
                    fileAccessWhitelist: FileAccessWhitelist,
                    directoryMembershipFingerprinter: this,
                    pathExpander: PathExpander,
                    executionLog: null,
                    fileSystemView: fileSystemView,
                    fileContentManager: new FileContentManager(this, new NullOperationTracker()),
                    directoryMembershipFinterprinterRuleSet: null);

                m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();

                ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
            }
Esempio n. 4
0
        /// <summary>
        /// Creates an instance of <see cref="GraphFragmentBuilder"/>.
        /// </summary>
        public GraphFragmentBuilderTopSort(LoggingContext loggingContext, PipExecutionContext pipExecutionContext, IConfiguration configuration, PathExpander pathExpander) : base(loggingContext, pipExecutionContext, configuration)
        {
            if (configuration.Schedule.ComputePipStaticFingerprints)
            {
                var extraFingerprintSalts = new ExtraFingerprintSalts(
                    configuration,
                    PipFingerprintingVersion.TwoPhaseV2,
                    configuration.Cache.CacheSalt,
                    new DirectoryMembershipFingerprinterRuleSet(configuration, pipExecutionContext.StringTable).ComputeSearchPathToolsHash());

                m_pipStaticFingerprinter = new PipStaticFingerprinter(
                    pipExecutionContext.PathTable,
                    GetSealDirectoryFingerprint,
                    GetDirectoryProducerFingerprint,
                    extraFingerprintSalts,
                    pathExpander)
                {
                    FingerprintTextEnabled = configuration.Schedule.LogPipStaticFingerprintTexts
                };
            }
        }
        /// <summary>
        /// Computes a fingerprint for looking up fingerprint store
        /// </summary>
        private static ContentFingerprint ComputeFingerprint(
            string key,
            IConfiguration configuration)
        {
            var extraFingerprintSalts = new ExtraFingerprintSalts(
                configuration,
                configuration.Cache.CacheSalt,
                null);

            using (var hasher = new BasicHashingHelper(recordFingerprintString: false))
            {
                hasher.Add("Type", "FingerprintStoreFingerprint");
                hasher.Add("FormatVersion", FingerprintStore.FormatVersion.Version.ToString());
                hasher.Add("LookupVersion", FingerprintStoreLookupVersion);
                hasher.Add("Key", key);
                hasher.Add("FingerprintSalt", extraFingerprintSalts.FingerprintSalt);

                var fingerprint = new ContentFingerprint(hasher.GenerateHash());
                return(fingerprint);
            }
        }
Esempio n. 6
0
        private static Optional <CompositeGraphFingerprint> GenerateHash(
            LoggingContext loggingContext,
            IStartupConfiguration startUpConfig,
            IConfiguration config,
            PathTable pathTable,
            IEvaluationFilter partialEvaluationData,
            FileContentTable fileContentTable,
            string commitId,
            EngineTestHooksData testHooks)
        {
            ILayoutConfiguration  layout  = config.Layout;
            ILoggingConfiguration logging = config.Logging;
            var fingerprintTextElements   = new List <(string, string)>();

            using (var hasher = new CoreHashingHelper(recordFingerprintString: true))
            {
                CompositeGraphFingerprint fingerprint = CompositeGraphFingerprint.Zero;

                AddInt(hasher, "version", GraphFingerprintVersion);

                using (var qualifierHasher = new CoreHashingHelper(recordFingerprintString: false))
                {
                    foreach (string qualifier in startUpConfig.QualifierIdentifiers.OrderBy(q => q))
                    {
                        AddText(qualifierHasher, "qualifier", qualifier);
                    }

                    fingerprint.QualifierHash = qualifierHasher.GenerateHash();
                    AddFingerprint(hasher, "Qualifiers", fingerprint.QualifierHash);
                }

                if (partialEvaluationData != null && partialEvaluationData.CanPerformPartialEvaluation)
                {
                    using (var evaluationFilterHasher = new CoreHashingHelper(recordFingerprintString: false))
                    {
                        foreach (string value in partialEvaluationData.ValueNamesToResolveAsStrings.OrderBy(v => v))
                        {
                            AddText(evaluationFilterHasher, "valueName", value);
                        }

                        foreach (string value in partialEvaluationData.ValueDefinitionRootsToResolveAsStrings.OrderBy(v => v))
                        {
                            AddText(evaluationFilterHasher, "valuePath", value);
                        }

                        foreach (string value in partialEvaluationData.ModulesToResolveAsStrings.OrderBy(v => v))
                        {
                            AddText(evaluationFilterHasher, "moduleName", value);
                        }

                        fingerprint.FilterHash = evaluationFilterHasher.GenerateHash();
                        AddFingerprint(hasher, "Values", fingerprint.FilterHash);
                    }
                }

                // These paths get embedded in the result of evaluation. So if any change we must re-evaluate
                AddText(hasher, "ObjectDirectoryPath", layout.ObjectDirectory.IsValid ? layout.ObjectDirectory.ToString(pathTable) : "::null::");
                AddText(hasher, "TempDirectoryPath", layout.TempDirectory.IsValid ? layout.TempDirectory.ToString(pathTable) : "::null::");
                AddText(hasher, "SourceDirectoryPath", layout.SourceDirectory.IsValid ? layout.SourceDirectory.ToString(pathTable) : "::null::");

                // All paths in the graph are relative to 'substTarget' (hence, 'substTarget' must be a part of the fingerprint, but 'substSource' need not be).
                AddText(hasher, "substTarget", logging.SubstTarget.IsValid ? logging.SubstTarget.ToString(pathTable) : "::null::");
                AddText(hasher, "IsCompressed", config.Engine.CompressGraphFiles ? "true" : "false");
                AddText(hasher, "IsSkipHashSourceFile", config.Schedule.SkipHashSourceFile ? "true" : "false");

                // Pip static fingerprints are not always computed because computing them slows down the graph construction by 10%-13%.
                // Thus, the pip graph may and may not contain pip static fingerprints. To avoid unexpected result due to graph cache hit,
                // we temporarily add the option for computing pip static fingerprints as part of our graph fingerprint until the fingerprints
                // are always computed; see Task 1291638.
                AddText(hasher, "ComputePipStaticFingerprints", config.Schedule.ComputePipStaticFingerprints.ToString());

                if (config.Schedule.ComputePipStaticFingerprints)
                {
                    // Pip static fingerprints are part of the graph and include the extra fingerprint salts.
                    // Thus, when pip static fingerprints are computed, any change to the salt will invalidate the graph because
                    // the pip static fingerprints will no longer be valid. Reusing the graph when the salt changes can result in
                    // underbuild.
                    var extraFingerprintSalts = new ExtraFingerprintSalts(
                        config,
                        PipFingerprintingVersion.TwoPhaseV2,
                        config.Cache.CacheSalt ?? string.Empty,
                        new Scheduler.DirectoryMembershipFingerprinterRuleSet(config, pathTable.StringTable).ComputeSearchPathToolsHash());

                    AddFingerprint(hasher, "ExtraFingerprintSalts", extraFingerprintSalts.CalculatedSaltsFingerprint);
                }

                // Config files
                // Caution: Including the content hash of the config file is how changes to the default pip filter
                // invalidate a cached graph. If the config file content hash is removed, the values that get
                // evaluated because of it must be reflected in the values passed in to this method.
                using (var configHasher = new CoreHashingHelper(recordFingerprintString: false))
                {
                    var configFiles = new List <AbsolutePath> {
                        startUpConfig.ConfigFile
                    };

                    try
                    {
                        foreach (var configPath in configFiles
                                 .Select(path => path.ToString(pathTable))
                                 .OrderBy(c => c, StringComparer.OrdinalIgnoreCase))
                        {
                            AddContentHash(
                                configHasher,
                                configPath.ToUpperInvariant(),
                                fileContentTable.GetAndRecordContentHashAsync(configPath)
                                .GetAwaiter()
                                .GetResult()
                                .VersionedFileIdentityAndContentInfo.FileContentInfo.Hash);
                        }

                        fingerprint.ConfigFileHash = configHasher.GenerateHash();
                        AddFingerprint(hasher, "ConfigFiles", fingerprint.ConfigFileHash);
                    }
                    catch (BuildXLException ex)
                    {
                        return(LogAndReturnFailure(ex));
                    }
                }

                if (!string.IsNullOrEmpty(commitId))
                {
                    using (var commitHasher = new CoreHashingHelper(recordFingerprintString: false))
                    {
                        commitHasher.Add("Commit", commitId);
                        fingerprint.BuildEngineHash = commitHasher.GenerateHash();
                    }

                    AddFingerprint(hasher, "BuildEngine", fingerprint.BuildEngineHash);
                }
                else
                {
                    // BuildXL assemblies. This will invalidate the cached graph if build files change
                    // or if the serialization format changes.
                    try
                    {
                        Action <string, ContentHash> handleBuildFileAndHash = (buildFile, buildFileHash) =>
                        {
                            // Directly add to fingerprint elements for logging, but the hash is represented in the build engine hash
                            fingerprintTextElements.Add((buildFile, buildFileHash.ToString()));
                        };

                        var deployment = testHooks?.AppDeployment ?? AppDeployment.ReadDeploymentManifestFromRunningApp();
                        fingerprint.BuildEngineHash = deployment.ComputeContentHashBasedFingerprint(fileContentTable, handleBuildFileAndHash);
                        AddFingerprint(hasher, "BuildEngine", fingerprint.BuildEngineHash);
                    }
                    catch (BuildXLException ex)
                    {
                        Tracing.Logger.Log.FailedToComputeHashFromDeploymentManifest(loggingContext);
                        Tracing.Logger.Log.FailedToComputeHashFromDeploymentManifestReason(loggingContext, ex.Message);
                        return(default(Optional <CompositeGraphFingerprint>));
                    }
                }

                AddText(hasher, "HostOS", startUpConfig.CurrentHost.CurrentOS.ToString());
                AddText(hasher, "HostCpuArchitecture", startUpConfig.CurrentHost.CpuArchitecture.ToString());
                AddText(hasher, "HostIsElevated", CurrentProcess.IsElevated.ToString());

                var salt = string.Empty;

                if (testHooks?.GraphFingerprintSalt != null)
                {
                    salt += testHooks.GraphFingerprintSalt.Value.ToString();
                }

                salt += EngineEnvironmentSettings.DebugGraphFingerprintSalt;

                if (!string.IsNullOrEmpty(salt))
                {
                    hasher.Add("GraphFingerprintSalt", salt);
                }

                fingerprint.OverallFingerprint = new ContentFingerprint(hasher.GenerateHash());
                Tracing.Logger.Log.ElementsOfConfigurationFingerprint(
                    loggingContext,
                    fingerprint.OverallFingerprint.ToString(),
                    string.Join(Environment.NewLine, fingerprintTextElements.Select(kvp => "\t" + kvp.Item1 + " : " + kvp.Item2)));

                return(new Optional <CompositeGraphFingerprint>(fingerprint));
            }

            // Local functions
            void AddInt(CoreHashingHelper hasher, string key, int value)
            {
                hasher.Add(key, value);
                fingerprintTextElements.Add(
                    (key, value.ToString(CultureInfo.InvariantCulture)));
            }

            void AddText(CoreHashingHelper hasher, string key, string value)
            {
                hasher.Add(key, value);
                fingerprintTextElements.Add((key, value));
            }

            void AddFingerprint(CoreHashingHelper hasher, string key, Fingerprint value)
            {
                hasher.Add(key, value);
                fingerprintTextElements.Add((key, value.ToString()));
            }

            void AddContentHash(CoreHashingHelper hasher, string key, ContentHash value)
            {
                hasher.Add(key, value);
                fingerprintTextElements.Add((key, value.ToString()));
            }
        }