Exemplo n.º 1
0
        /// <summary>
        /// Deserializes
        /// </summary>
        public static async Task <ConfigFileState> DeserializeAsync(
            BuildXLReader reader,
            LoggingContext loggingContext,
            Task <PipExecutionContext> contextTask)
        {
            Contract.Requires(reader != null);
            Contract.Requires(contextTask != null);

            FileAccessAllowlist allowlist = await FileAccessAllowlist.DeserializeAsync(reader, contextTask);

            string defaultFilter = reader.ReadString();
            string cacheSalt     = reader.ReadString();
            DirectoryMembershipFingerprinterRuleSet ruleSet = DirectoryMembershipFingerprinterRuleSet.Deserialize(reader);

            int moduleConfigurationsCount = reader.ReadInt32Compact();
            List <IModuleConfiguration> moduleConfigurations = new List <IModuleConfiguration>(moduleConfigurationsCount);

            for (int i = 0; i < moduleConfigurationsCount; i++)
            {
                var moduleConfiguration = new BuildXL.Utilities.Configuration.Mutable.ModuleConfiguration()
                {
                    ModuleId = reader.ReadModuleId(),
                    Name     = reader.ReadString(),
                };

                moduleConfigurations.Add(moduleConfiguration);
            }

            // TODO: Read everything else instead of doing it in many different places?
            return(new ConfigFileState(allowlist, defaultFilter, cacheSalt, ruleSet, moduleConfigurations));
        }
Exemplo n.º 2
0
        private static Scheduler CreateInternal(
            PipExecutionContext context,
            LoggingContext loggingContext,
            PipGraph pipGraph,
            IPipQueue queue,
            EngineCache cache,
            IConfiguration configuration)
        {
            Contract.Requires(context != null);
            Contract.Requires(queue != null);
            Contract.Requires(cache != null);
            Contract.Requires(configuration != null);

            var fileContentTable = FileContentTable.CreateNew(loggingContext);

            var fileAccessAllowList = new FileAccessAllowlist(context);

            var testHooks = new SchedulerTestHooks();

            return(new Scheduler(
                       pipGraph,
                       queue,
                       context,
                       fileContentTable,
                       cache: cache,
                       loggingContext: loggingContext,
                       configuration: configuration,
                       fileAccessAllowlist: fileAccessAllowList,
                       testHooks: testHooks,
                       buildEngineFingerprint: null,
                       tempCleaner: new TestMoveDeleteCleaner(Path.Combine(Environment.GetEnvironmentVariable("TEMP"), "MoveDeletionTemp"))));
        }
Exemplo n.º 3
0
        public async Task TestSerialization()
        {
            var context = BuildXLContext.CreateInstanceForTesting();

            var pathTable   = context.PathTable;
            var symbolTable = new SymbolTable(pathTable.StringTable);
            var allowlist   = new FileAccessAllowlist(context);

            var path1            = AbsolutePath.Create(pathTable, @"\\fakePath\foo.txt");
            var path2            = AbsolutePath.Create(pathTable, @"\\fakePath\bar.txt");
            var regex1           = new SerializableRegex(@"dir\foo.txt");
            var executableEntry1 = new ExecutablePathAllowlistEntry(
                path1, regex1, true, "entry1");
            var executableEntry2 = new ExecutablePathAllowlistEntry(
                path2, new SerializableRegex("bar"), false, "entry2");

            allowlist.Add(executableEntry1);
            allowlist.Add(executableEntry2);

            var symbol1    = FullSymbol.Create(symbolTable, "symbol1");
            var valueEntry = new ValuePathFileAccessAllowlistEntry(
                symbol1, new SerializableRegex("symbol1"), false, null);

            var symbol2     = FullSymbol.Create(symbolTable, "symbol2");
            var valueEntry2 = new ValuePathFileAccessAllowlistEntry(
                symbol2, new SerializableRegex("symbol2"), false, "entry4");

            allowlist.Add(valueEntry);
            allowlist.Add(valueEntry2);

            XAssert.AreEqual(3, allowlist.UncacheableEntryCount);
            XAssert.AreEqual(1, allowlist.CacheableEntryCount);
            XAssert.AreEqual("Unnamed", valueEntry.Name);

            using (var ms = new MemoryStream())
            {
                BuildXLWriter writer = new BuildXLWriter(true, ms, true, true);
                allowlist.Serialize(writer);

                ms.Position = 0;
                BuildXLReader reader       = new BuildXLReader(true, ms, true);
                var           deserialized = await FileAccessAllowlist.DeserializeAsync(reader, Task.FromResult <PipExecutionContext>(context));

                XAssert.AreEqual(2, deserialized.ExecutablePathEntries.Count);
                XAssert.AreEqual(1, deserialized.ExecutablePathEntries[path1].Count);
                XAssert.AreEqual(true, deserialized.ExecutablePathEntries[path1][0].AllowsCaching);
                XAssert.AreEqual(regex1.ToString(), deserialized.ExecutablePathEntries[path1][0].PathRegex.ToString());
                XAssert.AreEqual(executableEntry1.Name, deserialized.ExecutablePathEntries[path1][0].Name);
                XAssert.AreEqual(executableEntry2.Name, deserialized.ExecutablePathEntries[path2][0].Name);

                XAssert.AreEqual(2, deserialized.ValuePathEntries.Count);
                XAssert.AreEqual(1, deserialized.ValuePathEntries[symbol1].Count);
                XAssert.AreEqual(false, deserialized.ValuePathEntries[symbol1][0].AllowsCaching);
                XAssert.AreEqual(valueEntry.Name, deserialized.ValuePathEntries[symbol1][0].Name);
                XAssert.AreEqual(valueEntry2.Name, deserialized.ValuePathEntries[symbol2][0].Name);

                XAssert.AreEqual(3, deserialized.UncacheableEntryCount);
                XAssert.AreEqual(1, deserialized.CacheableEntryCount);
            }
        }
Exemplo n.º 4
0
        public TestScheduler(
            PipGraph graph,
            TestPipQueue pipQueue,
            PipExecutionContext context,
            FileContentTable fileContentTable,
            EngineCache cache,
            IConfiguration configuration,
            FileAccessAllowlist fileAccessAllowlist,
            DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules = null,
            ITempCleaner tempCleaner = null,
            HistoricPerfDataTable runningTimeTable = null,
            JournalState journalState = null,
            PerformanceCollector performanceCollector = null,
            string fingerprintSalt = null,
            PreserveOutputsInfo?previousInputsSalt = null,
            IEnumerable <Pip> successfulPips       = null,
            IEnumerable <Pip> failedPips           = null,
            LoggingContext loggingContext          = null,
            IIpcProvider ipcProvider = null,
            DirectoryTranslator directoryTranslator = null,
            VmInitializer vmInitializer             = null,
            SchedulerTestHooks testHooks            = null) : base(graph, pipQueue, context, fileContentTable, cache,
                                                                   configuration, fileAccessAllowlist, loggingContext, null, directoryMembershipFingerprinterRules,
                                                                   tempCleaner, AsyncLazy <HistoricPerfDataTable> .FromResult(runningTimeTable), performanceCollector, fingerprintSalt, previousInputsSalt,
                                                                   ipcProvider: ipcProvider,
                                                                   directoryTranslator: directoryTranslator,
                                                                   journalState: journalState,
                                                                   vmInitializer: vmInitializer,
                                                                   testHooks: testHooks)
        {
            m_testPipQueue = pipQueue;

            if (successfulPips != null)
            {
                foreach (var pip in successfulPips)
                {
                    Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler");
                    m_overridePipResults.Add(pip.PipId, PipResultStatus.Succeeded);
                }
            }

            if (failedPips != null)
            {
                foreach (var pip in failedPips)
                {
                    Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler");
                    m_overridePipResults.Add(pip.PipId, PipResultStatus.Failed);
                }
            }

            m_loggingContext = loggingContext;
        }
 /// <summary>
 /// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
 /// </summary>
 public DummyPipExecutionEnvironment(
     LoggingContext loggingContext,
     PipExecutionContext context,
     IConfiguration config,
     FileContentTable fileContentTable = null,
     EngineCache pipCache = null,
     SemanticPathExpander semanticPathExpander           = null,
     PipContentFingerprinter.PipDataLookup pipDataLookup = null,
     FileAccessAllowlist fileAccessAllowlist             = null,
     bool allowUnspecifiedSealedDirectories = false,
     PipTable pipTable        = null,
     IIpcProvider ipcProvider = null,
     (string substSource, string substTarget)?subst = default,
Exemplo n.º 6
0
 /// <summary>
 /// Constructor
 /// </summary>
 public ConfigFileState(
     FileAccessAllowlist fileAccessAllowlist,
     string defaultPipFilter,
     string cacheSalt,
     DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules,
     IList <IModuleConfiguration> moduleConfigurations)
 {
     FileAccessAllowlist = fileAccessAllowlist;
     DefaultPipFilter    = defaultPipFilter ?? string.Empty;
     CacheSalt           = cacheSalt ?? string.Empty;
     DirectoryMembershipFingerprinterRules = directoryMembershipFingerprinterRules;
     ModuleConfigurations = moduleConfigurations;
 }
Exemplo n.º 7
0
        /// <summary>
        /// Class constructor
        /// </summary>
        public PipExecutionState(
            IConfiguration configuration,
            LoggingContext loggingContext,
            PipTwoPhaseCache cache,
            FileAccessAllowlist fileAccessAllowlist,
            IDirectoryMembershipFingerprinter directoryMembershipFingerprinter,
            SemanticPathExpander pathExpander,
            IExecutionLogTarget executionLog,
            DirectoryMembershipFingerprinterRuleSet directoryMembershipFinterprinterRuleSet,
            FileContentManager fileContentManager,
            IUnsafeSandboxConfiguration unsafeConfiguration,
            PreserveOutputsInfo preserveOutputsSalt,
            FileSystemView fileSystemView,
            bool lazyDeletionOfSharedOpaqueOutputsEnabled,
            ServiceManager serviceManager = null)
        {
            Contract.Requires(fileContentManager != null);
            Contract.Requires(directoryMembershipFingerprinter != null);
            Contract.Requires(pathExpander != null);

            Cache = cache;
            m_fileAccessAllowlist            = fileAccessAllowlist;
            DirectoryMembershipFingerprinter = directoryMembershipFingerprinter;
            ResourceManager           = new ProcessResourceManager(loggingContext);
            m_pathExpander            = new FileContentManagerSemanticPathExpander(fileContentManager, pathExpander);
            ExecutionLog              = executionLog;
            m_rootModuleConfiguration = configuration;
            m_directoryMembershipFingerprinterRuleSet = directoryMembershipFinterprinterRuleSet;
            PathExistenceCache    = new ConcurrentBigMap <AbsolutePath, PathExistence>();
            FileContentManager    = fileContentManager;
            ServiceManager        = serviceManager ?? ServiceManager.Default;
            PipEnvironment        = new PipEnvironment(loggingContext);
            FileSystemView        = fileSystemView;
            m_unsafeConfiguration = unsafeConfiguration;
            m_preserveOutputsSalt = preserveOutputsSalt;
            LazyDeletionOfSharedOpaqueOutputsEnabled = lazyDeletionOfSharedOpaqueOutputsEnabled;

            if (fileSystemView != null)
            {
                fileContentManager.SetLocalDiskFileSystemExistenceView(fileSystemView);
            }
        }
        public async Task TestSerialization()
        {
            var context = BuildXLContext.CreateInstanceForTesting();

            var pathTable   = context.PathTable;
            var stringTable = context.StringTable;
            var symbolTable = new SymbolTable(pathTable.StringTable);
            var allowlist   = new FileAccessAllowlist(context);

            //Allowlist with full paths
            var path1            = new DiscriminatingUnion <AbsolutePath, PathAtom>(AbsolutePath.Create(pathTable, @"\\fakePath\foo.txt"));
            var path2            = new DiscriminatingUnion <AbsolutePath, PathAtom>(AbsolutePath.Create(pathTable, @"\\fakePath\bar.txt"));
            var regex1           = new SerializableRegex(@"dir\foo.txt");
            var executableEntry1 = new ExecutablePathAllowlistEntry(
                path1, regex1, true, "entry1");
            var executableEntry2 = new ExecutablePathAllowlistEntry(
                path2, new SerializableRegex("bar"), false, "entry2");

            allowlist.Add(executableEntry1);
            allowlist.Add(executableEntry2);

            // Allowlist with executable names only
            var path3            = new DiscriminatingUnion <AbsolutePath, PathAtom>(PathAtom.Create(stringTable, "alice.txt"));
            var path4            = new DiscriminatingUnion <AbsolutePath, PathAtom>(PathAtom.Create(stringTable, "bob.txt"));
            var regex3           = new SerializableRegex(@"dir\alice.txt");
            var executableEntry3 = new ExecutablePathAllowlistEntry(
                path3, regex3, true, "entry5");
            var executableEntry4 = new ExecutablePathAllowlistEntry(
                path4, new SerializableRegex("bob"), false, "entry6");

            allowlist.Add(executableEntry3);
            allowlist.Add(executableEntry4);

            var symbol1    = FullSymbol.Create(symbolTable, "symbol1");
            var valueEntry = new ValuePathFileAccessAllowlistEntry(
                symbol1, new SerializableRegex("symbol1"), false, null);

            var symbol2     = FullSymbol.Create(symbolTable, "symbol2");
            var valueEntry2 = new ValuePathFileAccessAllowlistEntry(
                symbol2, new SerializableRegex("symbol2"), false, "entry4");

            allowlist.Add(valueEntry);
            allowlist.Add(valueEntry2);

            XAssert.AreEqual(4, allowlist.UncacheableEntryCount);
            XAssert.AreEqual(2, allowlist.CacheableEntryCount);
            XAssert.AreEqual("Unnamed", valueEntry.Name);

            using (var ms = new MemoryStream())
            {
                BuildXLWriter writer = new BuildXLWriter(true, ms, true, true);
                allowlist.Serialize(writer);

                ms.Position = 0;
                BuildXLReader reader       = new BuildXLReader(true, ms, true);
                var           deserialized = await FileAccessAllowlist.DeserializeAsync(reader, Task.FromResult <PipExecutionContext>(context));

                var path1Absolute = (AbsolutePath)path1.GetValue();
                var path2Absolute = (AbsolutePath)path2.GetValue();
                var path3Atom     = ((PathAtom)path3.GetValue()).StringId;
                var path4Atom     = ((PathAtom)path4.GetValue()).StringId;

                XAssert.AreEqual(2, deserialized.ExecutablePathEntries.Count);
                XAssert.AreEqual(1, deserialized.ExecutablePathEntries[path1Absolute].Count);
                XAssert.AreEqual(true, deserialized.ExecutablePathEntries[path1Absolute][0].AllowsCaching);
                XAssert.AreEqual(regex1.ToString(), deserialized.ExecutablePathEntries[path1Absolute][0].PathRegex.ToString());
                XAssert.AreEqual(executableEntry1.Name, deserialized.ExecutablePathEntries[path1Absolute][0].Name);
                XAssert.AreEqual(executableEntry2.Name, deserialized.ExecutablePathEntries[path2Absolute][0].Name);

                XAssert.AreEqual(2, deserialized.ToolExecutableNameEntries.Count);
                XAssert.AreEqual(1, deserialized.ToolExecutableNameEntries[path3Atom].Count);
                XAssert.AreEqual(true, deserialized.ToolExecutableNameEntries[path3Atom][0].AllowsCaching);
                XAssert.AreEqual(regex3.ToString(), deserialized.ToolExecutableNameEntries[path3Atom][0].PathRegex.ToString());
                XAssert.AreEqual(executableEntry3.Name, deserialized.ToolExecutableNameEntries[path3Atom][0].Name);
                XAssert.AreEqual(executableEntry4.Name, deserialized.ToolExecutableNameEntries[path4Atom][0].Name);

                XAssert.AreEqual(2, deserialized.ValuePathEntries.Count);
                XAssert.AreEqual(1, deserialized.ValuePathEntries[symbol1].Count);
                XAssert.AreEqual(false, deserialized.ValuePathEntries[symbol1][0].AllowsCaching);
                XAssert.AreEqual(valueEntry.Name, deserialized.ValuePathEntries[symbol1][0].Name);
                XAssert.AreEqual(valueEntry2.Name, deserialized.ValuePathEntries[symbol2][0].Name);

                XAssert.AreEqual(4, deserialized.UncacheableEntryCount);
                XAssert.AreEqual(2, deserialized.CacheableEntryCount);
            }
        }