/// <summary>
/// Deserializes
/// </summary>
public static async Task <ConfigFileState> DeserializeAsync(
BuildXLReader reader,
LoggingContext loggingContext,
Task <PipExecutionContext> contextTask)
{
Contract.Requires(reader != null);
Contract.Requires(contextTask != null);
FileAccessAllowlist allowlist = await FileAccessAllowlist.DeserializeAsync(reader, contextTask);
string defaultFilter = reader.ReadString();
string cacheSalt = reader.ReadString();
DirectoryMembershipFingerprinterRuleSet ruleSet = DirectoryMembershipFingerprinterRuleSet.Deserialize(reader);
int moduleConfigurationsCount = reader.ReadInt32Compact();
List <IModuleConfiguration> moduleConfigurations = new List <IModuleConfiguration>(moduleConfigurationsCount);
for (int i = 0; i < moduleConfigurationsCount; i++)
{
var moduleConfiguration = new BuildXL.Utilities.Configuration.Mutable.ModuleConfiguration()
{
ModuleId = reader.ReadModuleId(),
Name = reader.ReadString(),
};
moduleConfigurations.Add(moduleConfiguration);
}
// TODO: Read everything else instead of doing it in many different places?
return(new ConfigFileState(allowlist, defaultFilter, cacheSalt, ruleSet, moduleConfigurations));
}
private static Scheduler CreateInternal(
PipExecutionContext context,
LoggingContext loggingContext,
PipGraph pipGraph,
IPipQueue queue,
EngineCache cache,
IConfiguration configuration)
{
Contract.Requires(context != null);
Contract.Requires(queue != null);
Contract.Requires(cache != null);
Contract.Requires(configuration != null);
var fileContentTable = FileContentTable.CreateNew(loggingContext);
var fileAccessAllowList = new FileAccessAllowlist(context);
var testHooks = new SchedulerTestHooks();
return(new Scheduler(
pipGraph,
queue,
context,
fileContentTable,
cache: cache,
loggingContext: loggingContext,
configuration: configuration,
fileAccessAllowlist: fileAccessAllowList,
testHooks: testHooks,
buildEngineFingerprint: null,
tempCleaner: new TestMoveDeleteCleaner(Path.Combine(Environment.GetEnvironmentVariable("TEMP"), "MoveDeletionTemp"))));
}
public async Task TestSerialization()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var symbolTable = new SymbolTable(pathTable.StringTable);
var allowlist = new FileAccessAllowlist(context);
var path1 = AbsolutePath.Create(pathTable, @"\\fakePath\foo.txt");
var path2 = AbsolutePath.Create(pathTable, @"\\fakePath\bar.txt");
var regex1 = new SerializableRegex(@"dir\foo.txt");
var executableEntry1 = new ExecutablePathAllowlistEntry(
path1, regex1, true, "entry1");
var executableEntry2 = new ExecutablePathAllowlistEntry(
path2, new SerializableRegex("bar"), false, "entry2");
allowlist.Add(executableEntry1);
allowlist.Add(executableEntry2);
var symbol1 = FullSymbol.Create(symbolTable, "symbol1");
var valueEntry = new ValuePathFileAccessAllowlistEntry(
symbol1, new SerializableRegex("symbol1"), false, null);
var symbol2 = FullSymbol.Create(symbolTable, "symbol2");
var valueEntry2 = new ValuePathFileAccessAllowlistEntry(
symbol2, new SerializableRegex("symbol2"), false, "entry4");
allowlist.Add(valueEntry);
allowlist.Add(valueEntry2);
XAssert.AreEqual(3, allowlist.UncacheableEntryCount);
XAssert.AreEqual(1, allowlist.CacheableEntryCount);
XAssert.AreEqual("Unnamed", valueEntry.Name);
using (var ms = new MemoryStream())
{
BuildXLWriter writer = new BuildXLWriter(true, ms, true, true);
allowlist.Serialize(writer);
ms.Position = 0;
BuildXLReader reader = new BuildXLReader(true, ms, true);
var deserialized = await FileAccessAllowlist.DeserializeAsync(reader, Task.FromResult <PipExecutionContext>(context));
XAssert.AreEqual(2, deserialized.ExecutablePathEntries.Count);
XAssert.AreEqual(1, deserialized.ExecutablePathEntries[path1].Count);
XAssert.AreEqual(true, deserialized.ExecutablePathEntries[path1][0].AllowsCaching);
XAssert.AreEqual(regex1.ToString(), deserialized.ExecutablePathEntries[path1][0].PathRegex.ToString());
XAssert.AreEqual(executableEntry1.Name, deserialized.ExecutablePathEntries[path1][0].Name);
XAssert.AreEqual(executableEntry2.Name, deserialized.ExecutablePathEntries[path2][0].Name);
XAssert.AreEqual(2, deserialized.ValuePathEntries.Count);
XAssert.AreEqual(1, deserialized.ValuePathEntries[symbol1].Count);
XAssert.AreEqual(false, deserialized.ValuePathEntries[symbol1][0].AllowsCaching);
XAssert.AreEqual(valueEntry.Name, deserialized.ValuePathEntries[symbol1][0].Name);
XAssert.AreEqual(valueEntry2.Name, deserialized.ValuePathEntries[symbol2][0].Name);
XAssert.AreEqual(3, deserialized.UncacheableEntryCount);
XAssert.AreEqual(1, deserialized.CacheableEntryCount);
}
}
public TestScheduler(
PipGraph graph,
TestPipQueue pipQueue,
PipExecutionContext context,
FileContentTable fileContentTable,
EngineCache cache,
IConfiguration configuration,
FileAccessAllowlist fileAccessAllowlist,
DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules = null,
ITempCleaner tempCleaner = null,
HistoricPerfDataTable runningTimeTable = null,
JournalState journalState = null,
PerformanceCollector performanceCollector = null,
string fingerprintSalt = null,
PreserveOutputsInfo?previousInputsSalt = null,
IEnumerable <Pip> successfulPips = null,
IEnumerable <Pip> failedPips = null,
LoggingContext loggingContext = null,
IIpcProvider ipcProvider = null,
DirectoryTranslator directoryTranslator = null,
VmInitializer vmInitializer = null,
SchedulerTestHooks testHooks = null) : base(graph, pipQueue, context, fileContentTable, cache,
configuration, fileAccessAllowlist, loggingContext, null, directoryMembershipFingerprinterRules,
tempCleaner, AsyncLazy <HistoricPerfDataTable> .FromResult(runningTimeTable), performanceCollector, fingerprintSalt, previousInputsSalt,
ipcProvider: ipcProvider,
directoryTranslator: directoryTranslator,
journalState: journalState,
vmInitializer: vmInitializer,
testHooks: testHooks)
{
m_testPipQueue = pipQueue;
if (successfulPips != null)
{
foreach (var pip in successfulPips)
{
Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler");
m_overridePipResults.Add(pip.PipId, PipResultStatus.Succeeded);
}
}
if (failedPips != null)
{
foreach (var pip in failedPips)
{
Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler");
m_overridePipResults.Add(pip.PipId, PipResultStatus.Failed);
}
}
m_loggingContext = loggingContext;
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessAllowlist fileAccessAllowlist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
(string substSource, string substTarget)?subst = default,
/// <summary>
/// Constructor
/// </summary>
public ConfigFileState(
FileAccessAllowlist fileAccessAllowlist,
string defaultPipFilter,
string cacheSalt,
DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules,
IList <IModuleConfiguration> moduleConfigurations)
{
FileAccessAllowlist = fileAccessAllowlist;
DefaultPipFilter = defaultPipFilter ?? string.Empty;
CacheSalt = cacheSalt ?? string.Empty;
DirectoryMembershipFingerprinterRules = directoryMembershipFingerprinterRules;
ModuleConfigurations = moduleConfigurations;
}
/// <summary>
/// Class constructor
/// </summary>
public PipExecutionState(
IConfiguration configuration,
LoggingContext loggingContext,
PipTwoPhaseCache cache,
FileAccessAllowlist fileAccessAllowlist,
IDirectoryMembershipFingerprinter directoryMembershipFingerprinter,
SemanticPathExpander pathExpander,
IExecutionLogTarget executionLog,
DirectoryMembershipFingerprinterRuleSet directoryMembershipFinterprinterRuleSet,
FileContentManager fileContentManager,
IUnsafeSandboxConfiguration unsafeConfiguration,
PreserveOutputsInfo preserveOutputsSalt,
FileSystemView fileSystemView,
bool lazyDeletionOfSharedOpaqueOutputsEnabled,
ServiceManager serviceManager = null)
{
Contract.Requires(fileContentManager != null);
Contract.Requires(directoryMembershipFingerprinter != null);
Contract.Requires(pathExpander != null);
Cache = cache;
m_fileAccessAllowlist = fileAccessAllowlist;
DirectoryMembershipFingerprinter = directoryMembershipFingerprinter;
ResourceManager = new ProcessResourceManager(loggingContext);
m_pathExpander = new FileContentManagerSemanticPathExpander(fileContentManager, pathExpander);
ExecutionLog = executionLog;
m_rootModuleConfiguration = configuration;
m_directoryMembershipFingerprinterRuleSet = directoryMembershipFinterprinterRuleSet;
PathExistenceCache = new ConcurrentBigMap <AbsolutePath, PathExistence>();
FileContentManager = fileContentManager;
ServiceManager = serviceManager ?? ServiceManager.Default;
PipEnvironment = new PipEnvironment(loggingContext);
FileSystemView = fileSystemView;
m_unsafeConfiguration = unsafeConfiguration;
m_preserveOutputsSalt = preserveOutputsSalt;
LazyDeletionOfSharedOpaqueOutputsEnabled = lazyDeletionOfSharedOpaqueOutputsEnabled;
if (fileSystemView != null)
{
fileContentManager.SetLocalDiskFileSystemExistenceView(fileSystemView);
}
}
public async Task TestSerialization()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var stringTable = context.StringTable;
var symbolTable = new SymbolTable(pathTable.StringTable);
var allowlist = new FileAccessAllowlist(context);
//Allowlist with full paths
var path1 = new DiscriminatingUnion <AbsolutePath, PathAtom>(AbsolutePath.Create(pathTable, @"\\fakePath\foo.txt"));
var path2 = new DiscriminatingUnion <AbsolutePath, PathAtom>(AbsolutePath.Create(pathTable, @"\\fakePath\bar.txt"));
var regex1 = new SerializableRegex(@"dir\foo.txt");
var executableEntry1 = new ExecutablePathAllowlistEntry(
path1, regex1, true, "entry1");
var executableEntry2 = new ExecutablePathAllowlistEntry(
path2, new SerializableRegex("bar"), false, "entry2");
allowlist.Add(executableEntry1);
allowlist.Add(executableEntry2);
// Allowlist with executable names only
var path3 = new DiscriminatingUnion <AbsolutePath, PathAtom>(PathAtom.Create(stringTable, "alice.txt"));
var path4 = new DiscriminatingUnion <AbsolutePath, PathAtom>(PathAtom.Create(stringTable, "bob.txt"));
var regex3 = new SerializableRegex(@"dir\alice.txt");
var executableEntry3 = new ExecutablePathAllowlistEntry(
path3, regex3, true, "entry5");
var executableEntry4 = new ExecutablePathAllowlistEntry(
path4, new SerializableRegex("bob"), false, "entry6");
allowlist.Add(executableEntry3);
allowlist.Add(executableEntry4);
var symbol1 = FullSymbol.Create(symbolTable, "symbol1");
var valueEntry = new ValuePathFileAccessAllowlistEntry(
symbol1, new SerializableRegex("symbol1"), false, null);
var symbol2 = FullSymbol.Create(symbolTable, "symbol2");
var valueEntry2 = new ValuePathFileAccessAllowlistEntry(
symbol2, new SerializableRegex("symbol2"), false, "entry4");
allowlist.Add(valueEntry);
allowlist.Add(valueEntry2);
XAssert.AreEqual(4, allowlist.UncacheableEntryCount);
XAssert.AreEqual(2, allowlist.CacheableEntryCount);
XAssert.AreEqual("Unnamed", valueEntry.Name);
using (var ms = new MemoryStream())
{
BuildXLWriter writer = new BuildXLWriter(true, ms, true, true);
allowlist.Serialize(writer);
ms.Position = 0;
BuildXLReader reader = new BuildXLReader(true, ms, true);
var deserialized = await FileAccessAllowlist.DeserializeAsync(reader, Task.FromResult <PipExecutionContext>(context));
var path1Absolute = (AbsolutePath)path1.GetValue();
var path2Absolute = (AbsolutePath)path2.GetValue();
var path3Atom = ((PathAtom)path3.GetValue()).StringId;
var path4Atom = ((PathAtom)path4.GetValue()).StringId;
XAssert.AreEqual(2, deserialized.ExecutablePathEntries.Count);
XAssert.AreEqual(1, deserialized.ExecutablePathEntries[path1Absolute].Count);
XAssert.AreEqual(true, deserialized.ExecutablePathEntries[path1Absolute][0].AllowsCaching);
XAssert.AreEqual(regex1.ToString(), deserialized.ExecutablePathEntries[path1Absolute][0].PathRegex.ToString());
XAssert.AreEqual(executableEntry1.Name, deserialized.ExecutablePathEntries[path1Absolute][0].Name);
XAssert.AreEqual(executableEntry2.Name, deserialized.ExecutablePathEntries[path2Absolute][0].Name);
XAssert.AreEqual(2, deserialized.ToolExecutableNameEntries.Count);
XAssert.AreEqual(1, deserialized.ToolExecutableNameEntries[path3Atom].Count);
XAssert.AreEqual(true, deserialized.ToolExecutableNameEntries[path3Atom][0].AllowsCaching);
XAssert.AreEqual(regex3.ToString(), deserialized.ToolExecutableNameEntries[path3Atom][0].PathRegex.ToString());
XAssert.AreEqual(executableEntry3.Name, deserialized.ToolExecutableNameEntries[path3Atom][0].Name);
XAssert.AreEqual(executableEntry4.Name, deserialized.ToolExecutableNameEntries[path4Atom][0].Name);
XAssert.AreEqual(2, deserialized.ValuePathEntries.Count);
XAssert.AreEqual(1, deserialized.ValuePathEntries[symbol1].Count);
XAssert.AreEqual(false, deserialized.ValuePathEntries[symbol1][0].AllowsCaching);
XAssert.AreEqual(valueEntry.Name, deserialized.ValuePathEntries[symbol1][0].Name);
XAssert.AreEqual(valueEntry2.Name, deserialized.ValuePathEntries[symbol2][0].Name);
XAssert.AreEqual(4, deserialized.UncacheableEntryCount);
XAssert.AreEqual(2, deserialized.CacheableEntryCount);
}
}
