/// <summary> /// Deserializes /// </summary> public static async Task <ConfigFileState> DeserializeAsync( BuildXLReader reader, LoggingContext loggingContext, Task <PipExecutionContext> contextTask) { Contract.Requires(reader != null); Contract.Requires(contextTask != null); FileAccessAllowlist allowlist = await FileAccessAllowlist.DeserializeAsync(reader, contextTask); string defaultFilter = reader.ReadString(); string cacheSalt = reader.ReadString(); DirectoryMembershipFingerprinterRuleSet ruleSet = DirectoryMembershipFingerprinterRuleSet.Deserialize(reader); int moduleConfigurationsCount = reader.ReadInt32Compact(); List <IModuleConfiguration> moduleConfigurations = new List <IModuleConfiguration>(moduleConfigurationsCount); for (int i = 0; i < moduleConfigurationsCount; i++) { var moduleConfiguration = new BuildXL.Utilities.Configuration.Mutable.ModuleConfiguration() { ModuleId = reader.ReadModuleId(), Name = reader.ReadString(), }; moduleConfigurations.Add(moduleConfiguration); } // TODO: Read everything else instead of doing it in many different places? return(new ConfigFileState(allowlist, defaultFilter, cacheSalt, ruleSet, moduleConfigurations)); }
private static Scheduler CreateInternal( PipExecutionContext context, LoggingContext loggingContext, PipGraph pipGraph, IPipQueue queue, EngineCache cache, IConfiguration configuration) { Contract.Requires(context != null); Contract.Requires(queue != null); Contract.Requires(cache != null); Contract.Requires(configuration != null); var fileContentTable = FileContentTable.CreateNew(loggingContext); var fileAccessAllowList = new FileAccessAllowlist(context); var testHooks = new SchedulerTestHooks(); return(new Scheduler( pipGraph, queue, context, fileContentTable, cache: cache, loggingContext: loggingContext, configuration: configuration, fileAccessAllowlist: fileAccessAllowList, testHooks: testHooks, buildEngineFingerprint: null, tempCleaner: new TestMoveDeleteCleaner(Path.Combine(Environment.GetEnvironmentVariable("TEMP"), "MoveDeletionTemp")))); }
public async Task TestSerialization() { var context = BuildXLContext.CreateInstanceForTesting(); var pathTable = context.PathTable; var symbolTable = new SymbolTable(pathTable.StringTable); var allowlist = new FileAccessAllowlist(context); var path1 = AbsolutePath.Create(pathTable, @"\\fakePath\foo.txt"); var path2 = AbsolutePath.Create(pathTable, @"\\fakePath\bar.txt"); var regex1 = new SerializableRegex(@"dir\foo.txt"); var executableEntry1 = new ExecutablePathAllowlistEntry( path1, regex1, true, "entry1"); var executableEntry2 = new ExecutablePathAllowlistEntry( path2, new SerializableRegex("bar"), false, "entry2"); allowlist.Add(executableEntry1); allowlist.Add(executableEntry2); var symbol1 = FullSymbol.Create(symbolTable, "symbol1"); var valueEntry = new ValuePathFileAccessAllowlistEntry( symbol1, new SerializableRegex("symbol1"), false, null); var symbol2 = FullSymbol.Create(symbolTable, "symbol2"); var valueEntry2 = new ValuePathFileAccessAllowlistEntry( symbol2, new SerializableRegex("symbol2"), false, "entry4"); allowlist.Add(valueEntry); allowlist.Add(valueEntry2); XAssert.AreEqual(3, allowlist.UncacheableEntryCount); XAssert.AreEqual(1, allowlist.CacheableEntryCount); XAssert.AreEqual("Unnamed", valueEntry.Name); using (var ms = new MemoryStream()) { BuildXLWriter writer = new BuildXLWriter(true, ms, true, true); allowlist.Serialize(writer); ms.Position = 0; BuildXLReader reader = new BuildXLReader(true, ms, true); var deserialized = await FileAccessAllowlist.DeserializeAsync(reader, Task.FromResult <PipExecutionContext>(context)); XAssert.AreEqual(2, deserialized.ExecutablePathEntries.Count); XAssert.AreEqual(1, deserialized.ExecutablePathEntries[path1].Count); XAssert.AreEqual(true, deserialized.ExecutablePathEntries[path1][0].AllowsCaching); XAssert.AreEqual(regex1.ToString(), deserialized.ExecutablePathEntries[path1][0].PathRegex.ToString()); XAssert.AreEqual(executableEntry1.Name, deserialized.ExecutablePathEntries[path1][0].Name); XAssert.AreEqual(executableEntry2.Name, deserialized.ExecutablePathEntries[path2][0].Name); XAssert.AreEqual(2, deserialized.ValuePathEntries.Count); XAssert.AreEqual(1, deserialized.ValuePathEntries[symbol1].Count); XAssert.AreEqual(false, deserialized.ValuePathEntries[symbol1][0].AllowsCaching); XAssert.AreEqual(valueEntry.Name, deserialized.ValuePathEntries[symbol1][0].Name); XAssert.AreEqual(valueEntry2.Name, deserialized.ValuePathEntries[symbol2][0].Name); XAssert.AreEqual(3, deserialized.UncacheableEntryCount); XAssert.AreEqual(1, deserialized.CacheableEntryCount); } }
public TestScheduler( PipGraph graph, TestPipQueue pipQueue, PipExecutionContext context, FileContentTable fileContentTable, EngineCache cache, IConfiguration configuration, FileAccessAllowlist fileAccessAllowlist, DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules = null, ITempCleaner tempCleaner = null, HistoricPerfDataTable runningTimeTable = null, JournalState journalState = null, PerformanceCollector performanceCollector = null, string fingerprintSalt = null, PreserveOutputsInfo?previousInputsSalt = null, IEnumerable <Pip> successfulPips = null, IEnumerable <Pip> failedPips = null, LoggingContext loggingContext = null, IIpcProvider ipcProvider = null, DirectoryTranslator directoryTranslator = null, VmInitializer vmInitializer = null, SchedulerTestHooks testHooks = null) : base(graph, pipQueue, context, fileContentTable, cache, configuration, fileAccessAllowlist, loggingContext, null, directoryMembershipFingerprinterRules, tempCleaner, AsyncLazy <HistoricPerfDataTable> .FromResult(runningTimeTable), performanceCollector, fingerprintSalt, previousInputsSalt, ipcProvider: ipcProvider, directoryTranslator: directoryTranslator, journalState: journalState, vmInitializer: vmInitializer, testHooks: testHooks) { m_testPipQueue = pipQueue; if (successfulPips != null) { foreach (var pip in successfulPips) { Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler"); m_overridePipResults.Add(pip.PipId, PipResultStatus.Succeeded); } } if (failedPips != null) { foreach (var pip in failedPips) { Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler"); m_overridePipResults.Add(pip.PipId, PipResultStatus.Failed); } } m_loggingContext = loggingContext; }
/// <summary> /// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified. /// </summary> public DummyPipExecutionEnvironment( LoggingContext loggingContext, PipExecutionContext context, IConfiguration config, FileContentTable fileContentTable = null, EngineCache pipCache = null, SemanticPathExpander semanticPathExpander = null, PipContentFingerprinter.PipDataLookup pipDataLookup = null, FileAccessAllowlist fileAccessAllowlist = null, bool allowUnspecifiedSealedDirectories = false, PipTable pipTable = null, IIpcProvider ipcProvider = null, (string substSource, string substTarget)?subst = default,
/// <summary> /// Constructor /// </summary> public ConfigFileState( FileAccessAllowlist fileAccessAllowlist, string defaultPipFilter, string cacheSalt, DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules, IList <IModuleConfiguration> moduleConfigurations) { FileAccessAllowlist = fileAccessAllowlist; DefaultPipFilter = defaultPipFilter ?? string.Empty; CacheSalt = cacheSalt ?? string.Empty; DirectoryMembershipFingerprinterRules = directoryMembershipFingerprinterRules; ModuleConfigurations = moduleConfigurations; }
/// <summary> /// Class constructor /// </summary> public PipExecutionState( IConfiguration configuration, LoggingContext loggingContext, PipTwoPhaseCache cache, FileAccessAllowlist fileAccessAllowlist, IDirectoryMembershipFingerprinter directoryMembershipFingerprinter, SemanticPathExpander pathExpander, IExecutionLogTarget executionLog, DirectoryMembershipFingerprinterRuleSet directoryMembershipFinterprinterRuleSet, FileContentManager fileContentManager, IUnsafeSandboxConfiguration unsafeConfiguration, PreserveOutputsInfo preserveOutputsSalt, FileSystemView fileSystemView, bool lazyDeletionOfSharedOpaqueOutputsEnabled, ServiceManager serviceManager = null) { Contract.Requires(fileContentManager != null); Contract.Requires(directoryMembershipFingerprinter != null); Contract.Requires(pathExpander != null); Cache = cache; m_fileAccessAllowlist = fileAccessAllowlist; DirectoryMembershipFingerprinter = directoryMembershipFingerprinter; ResourceManager = new ProcessResourceManager(loggingContext); m_pathExpander = new FileContentManagerSemanticPathExpander(fileContentManager, pathExpander); ExecutionLog = executionLog; m_rootModuleConfiguration = configuration; m_directoryMembershipFingerprinterRuleSet = directoryMembershipFinterprinterRuleSet; PathExistenceCache = new ConcurrentBigMap <AbsolutePath, PathExistence>(); FileContentManager = fileContentManager; ServiceManager = serviceManager ?? ServiceManager.Default; PipEnvironment = new PipEnvironment(loggingContext); FileSystemView = fileSystemView; m_unsafeConfiguration = unsafeConfiguration; m_preserveOutputsSalt = preserveOutputsSalt; LazyDeletionOfSharedOpaqueOutputsEnabled = lazyDeletionOfSharedOpaqueOutputsEnabled; if (fileSystemView != null) { fileContentManager.SetLocalDiskFileSystemExistenceView(fileSystemView); } }
public async Task TestSerialization() { var context = BuildXLContext.CreateInstanceForTesting(); var pathTable = context.PathTable; var stringTable = context.StringTable; var symbolTable = new SymbolTable(pathTable.StringTable); var allowlist = new FileAccessAllowlist(context); //Allowlist with full paths var path1 = new DiscriminatingUnion <AbsolutePath, PathAtom>(AbsolutePath.Create(pathTable, @"\\fakePath\foo.txt")); var path2 = new DiscriminatingUnion <AbsolutePath, PathAtom>(AbsolutePath.Create(pathTable, @"\\fakePath\bar.txt")); var regex1 = new SerializableRegex(@"dir\foo.txt"); var executableEntry1 = new ExecutablePathAllowlistEntry( path1, regex1, true, "entry1"); var executableEntry2 = new ExecutablePathAllowlistEntry( path2, new SerializableRegex("bar"), false, "entry2"); allowlist.Add(executableEntry1); allowlist.Add(executableEntry2); // Allowlist with executable names only var path3 = new DiscriminatingUnion <AbsolutePath, PathAtom>(PathAtom.Create(stringTable, "alice.txt")); var path4 = new DiscriminatingUnion <AbsolutePath, PathAtom>(PathAtom.Create(stringTable, "bob.txt")); var regex3 = new SerializableRegex(@"dir\alice.txt"); var executableEntry3 = new ExecutablePathAllowlistEntry( path3, regex3, true, "entry5"); var executableEntry4 = new ExecutablePathAllowlistEntry( path4, new SerializableRegex("bob"), false, "entry6"); allowlist.Add(executableEntry3); allowlist.Add(executableEntry4); var symbol1 = FullSymbol.Create(symbolTable, "symbol1"); var valueEntry = new ValuePathFileAccessAllowlistEntry( symbol1, new SerializableRegex("symbol1"), false, null); var symbol2 = FullSymbol.Create(symbolTable, "symbol2"); var valueEntry2 = new ValuePathFileAccessAllowlistEntry( symbol2, new SerializableRegex("symbol2"), false, "entry4"); allowlist.Add(valueEntry); allowlist.Add(valueEntry2); XAssert.AreEqual(4, allowlist.UncacheableEntryCount); XAssert.AreEqual(2, allowlist.CacheableEntryCount); XAssert.AreEqual("Unnamed", valueEntry.Name); using (var ms = new MemoryStream()) { BuildXLWriter writer = new BuildXLWriter(true, ms, true, true); allowlist.Serialize(writer); ms.Position = 0; BuildXLReader reader = new BuildXLReader(true, ms, true); var deserialized = await FileAccessAllowlist.DeserializeAsync(reader, Task.FromResult <PipExecutionContext>(context)); var path1Absolute = (AbsolutePath)path1.GetValue(); var path2Absolute = (AbsolutePath)path2.GetValue(); var path3Atom = ((PathAtom)path3.GetValue()).StringId; var path4Atom = ((PathAtom)path4.GetValue()).StringId; XAssert.AreEqual(2, deserialized.ExecutablePathEntries.Count); XAssert.AreEqual(1, deserialized.ExecutablePathEntries[path1Absolute].Count); XAssert.AreEqual(true, deserialized.ExecutablePathEntries[path1Absolute][0].AllowsCaching); XAssert.AreEqual(regex1.ToString(), deserialized.ExecutablePathEntries[path1Absolute][0].PathRegex.ToString()); XAssert.AreEqual(executableEntry1.Name, deserialized.ExecutablePathEntries[path1Absolute][0].Name); XAssert.AreEqual(executableEntry2.Name, deserialized.ExecutablePathEntries[path2Absolute][0].Name); XAssert.AreEqual(2, deserialized.ToolExecutableNameEntries.Count); XAssert.AreEqual(1, deserialized.ToolExecutableNameEntries[path3Atom].Count); XAssert.AreEqual(true, deserialized.ToolExecutableNameEntries[path3Atom][0].AllowsCaching); XAssert.AreEqual(regex3.ToString(), deserialized.ToolExecutableNameEntries[path3Atom][0].PathRegex.ToString()); XAssert.AreEqual(executableEntry3.Name, deserialized.ToolExecutableNameEntries[path3Atom][0].Name); XAssert.AreEqual(executableEntry4.Name, deserialized.ToolExecutableNameEntries[path4Atom][0].Name); XAssert.AreEqual(2, deserialized.ValuePathEntries.Count); XAssert.AreEqual(1, deserialized.ValuePathEntries[symbol1].Count); XAssert.AreEqual(false, deserialized.ValuePathEntries[symbol1][0].AllowsCaching); XAssert.AreEqual(valueEntry.Name, deserialized.ValuePathEntries[symbol1][0].Name); XAssert.AreEqual(valueEntry2.Name, deserialized.ValuePathEntries[symbol2][0].Name); XAssert.AreEqual(4, deserialized.UncacheableEntryCount); XAssert.AreEqual(2, deserialized.CacheableEntryCount); } }