private static Scheduler CreateInternal(
PipExecutionContext context,
PipGraph pipGraph,
IPipQueue queue,
EngineCache cache,
IConfiguration configuration)
{
Contract.Requires(context != null);
Contract.Requires(queue != null);
Contract.Requires(cache != null);
Contract.Requires(configuration != null);
var fileContentTable = FileContentTable.CreateNew();
var fileAccessWhiteList = new FileAccessWhitelist(context);
var testHooks = new SchedulerTestHooks();
return(new Scheduler(
pipGraph,
queue,
context,
fileContentTable,
cache: cache,
loggingContext: Events.StaticContext,
configuration: configuration,
fileAccessWhitelist: fileAccessWhiteList,
testHooks: testHooks,
buildEngineFingerprint: null));
}
private static Scheduler CreateInternal(
PipExecutionContext context,
PipGraph pipGraph,
IPipQueue queue,
EngineCache cache,
IConfiguration configuration)
{
Contract.Requires(context != null);
Contract.Requires(queue != null);
Contract.Requires(cache != null);
Contract.Requires(configuration != null);
var fileContentTable = FileContentTable.CreateNew();
var fileAccessWhiteList = new FileAccessWhitelist(context);
var testHooks = new SchedulerTestHooks();
return(new Scheduler(
pipGraph,
queue,
context,
fileContentTable,
cache: cache,
loggingContext: Events.StaticContext,
configuration: configuration,
fileAccessWhitelist: fileAccessWhiteList,
testHooks: testHooks,
buildEngineFingerprint: null,
tempCleaner: new TestMoveDeleteCleaner(Path.Combine(Environment.GetEnvironmentVariable("TEMP"), "MoveDeletionTemp"))));
}
public async Task TestSerialization()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var symbolTable = new SymbolTable(pathTable.StringTable);
var whitelist = new FileAccessWhitelist(context);
var path1 = AbsolutePath.Create(pathTable, @"\\fakePath\foo.txt");
var path2 = AbsolutePath.Create(pathTable, @"\\fakePath\bar.txt");
var regex1 = new SerializableRegex(@"dir\foo.txt");
var executableEntry1 = new ExecutablePathWhitelistEntry(
path1, regex1, true, "entry1");
var executableEntry2 = new ExecutablePathWhitelistEntry(
path2, new SerializableRegex("bar"), false, "entry2");
whitelist.Add(executableEntry1);
whitelist.Add(executableEntry2);
var symbol1 = FullSymbol.Create(symbolTable, "symbol1");
var valueEntry = new ValuePathFileAccessWhitelistEntry(
symbol1, new SerializableRegex("symbol1"), false, null);
var symbol2 = FullSymbol.Create(symbolTable, "symbol2");
var valueEntry2 = new ValuePathFileAccessWhitelistEntry(
symbol2, new SerializableRegex("symbol2"), false, "entry4");
whitelist.Add(valueEntry);
whitelist.Add(valueEntry2);
XAssert.AreEqual(3, whitelist.UncacheableEntryCount);
XAssert.AreEqual(1, whitelist.CacheableEntryCount);
XAssert.AreEqual("Unnamed", valueEntry.Name);
using (var ms = new MemoryStream())
{
BuildXLWriter writer = new BuildXLWriter(true, ms, true, true);
whitelist.Serialize(writer);
ms.Position = 0;
BuildXLReader reader = new BuildXLReader(true, ms, true);
var deserialized = await FileAccessWhitelist.DeserializeAsync(reader, Task.FromResult <PipExecutionContext>(context));
XAssert.AreEqual(2, deserialized.ExecutablePathEntries.Count);
XAssert.AreEqual(1, deserialized.ExecutablePathEntries[path1].Count);
XAssert.AreEqual(true, deserialized.ExecutablePathEntries[path1][0].AllowsCaching);
XAssert.AreEqual(regex1.ToString(), deserialized.ExecutablePathEntries[path1][0].PathRegex.ToString());
XAssert.AreEqual(executableEntry1.Name, deserialized.ExecutablePathEntries[path1][0].Name);
XAssert.AreEqual(executableEntry2.Name, deserialized.ExecutablePathEntries[path2][0].Name);
XAssert.AreEqual(2, deserialized.ValuePathEntries.Count);
XAssert.AreEqual(1, deserialized.ValuePathEntries[symbol1].Count);
XAssert.AreEqual(false, deserialized.ValuePathEntries[symbol1][0].AllowsCaching);
XAssert.AreEqual(valueEntry.Name, deserialized.ValuePathEntries[symbol1][0].Name);
XAssert.AreEqual(valueEntry2.Name, deserialized.ValuePathEntries[symbol2][0].Name);
XAssert.AreEqual(3, deserialized.UncacheableEntryCount);
XAssert.AreEqual(1, deserialized.CacheableEntryCount);
}
}
/// <summary>
/// Deserializes
/// </summary>
public static async Task <ConfigFileState> DeserializeAsync(
BuildXLReader reader,
Task <PipExecutionContext> contextTask)
{
Contract.Requires(reader != null);
Contract.Requires(contextTask != null);
FileAccessWhitelist whitelist = await FileAccessWhitelist.DeserializeAsync(reader, contextTask);
string defaultFilter = reader.ReadString();
string cacheSalt = reader.ReadString();
DirectoryMembershipFingerprinterRuleSet ruleSet = DirectoryMembershipFingerprinterRuleSet.Deserialize(reader);
int moduleConfigurationsCount = reader.ReadInt32Compact();
List <IModuleConfiguration> moduleConfigurations = new List <IModuleConfiguration>(moduleConfigurationsCount);
for (int i = 0; i < moduleConfigurationsCount; i++)
{
var moduleConfiguration = new BuildXL.Utilities.Configuration.Mutable.ModuleConfiguration()
{
ModuleId = reader.ReadModuleId(),
Name = reader.ReadString(),
};
moduleConfigurations.Add(moduleConfiguration);
}
// TODO: Read everything else instead of doing it in many different places?
return(new ConfigFileState(whitelist, defaultFilter, cacheSalt, ruleSet, moduleConfigurations));
}
public TestScheduler(
PipGraph graph,
TestPipQueue pipQueue,
PipExecutionContext context,
FileContentTable fileContentTable,
EngineCache cache,
IConfiguration configuration,
FileAccessWhitelist fileAccessWhitelist,
DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules = null,
ITempCleaner tempCleaner = null,
PipRuntimeTimeTable runningTimeTable = null,
JournalState journalState = null,
PerformanceCollector performanceCollector = null,
string fingerprintSalt = null,
PreserveOutputsInfo?previousInputsSalt = null,
IEnumerable <Pip> successfulPips = null,
IEnumerable <Pip> failedPips = null,
LoggingContext loggingContext = null,
IIpcProvider ipcProvider = null,
DirectoryTranslator directoryTranslator = null,
VmInitializer vmInitializer = null,
SchedulerTestHooks testHooks = null) : base(graph, pipQueue, context, fileContentTable, cache,
configuration, fileAccessWhitelist, loggingContext, null, directoryMembershipFingerprinterRules,
tempCleaner, AsyncLazy <PipRuntimeTimeTable> .FromResult(runningTimeTable), performanceCollector, fingerprintSalt, previousInputsSalt,
ipcProvider: ipcProvider,
directoryTranslator: directoryTranslator,
journalState: journalState,
vmInitializer: vmInitializer,
testHooks: testHooks)
{
m_testPipQueue = pipQueue;
if (successfulPips != null)
{
foreach (var pip in successfulPips)
{
Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler");
m_overridePipResults.Add(pip.PipId, PipResultStatus.Succeeded);
}
}
if (failedPips != null)
{
foreach (var pip in failedPips)
{
Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler");
m_overridePipResults.Add(pip.PipId, PipResultStatus.Failed);
}
}
m_loggingContext = loggingContext;
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
(string substSource, string substTarget)?subst = default,
/// <summary>
/// Constructor
/// </summary>
public ConfigFileState(
FileAccessWhitelist fileAccessWhitelist,
string defaultPipFilter,
string cacheSalt,
DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules,
IList <IModuleConfiguration> moduleConfigurations)
{
FileAccessWhitelist = fileAccessWhitelist;
DefaultPipFilter = defaultPipFilter ?? string.Empty;
CacheSalt = cacheSalt ?? string.Empty;
DirectoryMembershipFingerprinterRules = directoryMembershipFingerprinterRules;
ModuleConfigurations = moduleConfigurations;
}
/// <summary>
/// Class constructor
/// </summary>
public PipExecutionState(
IConfiguration configuration,
LoggingContext loggingContext,
PipTwoPhaseCache cache,
FileAccessWhitelist fileAccessWhitelist,
IDirectoryMembershipFingerprinter directoryMembershipFingerprinter,
SemanticPathExpander pathExpander,
IExecutionLogTarget executionLog,
DirectoryMembershipFingerprinterRuleSet directoryMembershipFinterprinterRuleSet,
FileContentManager fileContentManager,
IUnsafeSandboxConfiguration unsafeConfiguration,
PreserveOutputsInfo preserveOutputsSalt,
FileSystemView fileSystemView,
bool lazyDeletionOfSharedOpaqueOutputsEnabled,
ServiceManager serviceManager = null)
{
Contract.Requires(fileContentManager != null);
Contract.Requires(directoryMembershipFingerprinter != null);
Contract.Requires(pathExpander != null);
Cache = cache;
m_fileAccessWhitelist = fileAccessWhitelist;
DirectoryMembershipFingerprinter = directoryMembershipFingerprinter;
ResourceManager = new ProcessResourceManager(loggingContext);
m_pathExpander = new FileContentManagerSemanticPathExpander(fileContentManager, pathExpander);
ExecutionLog = executionLog;
m_rootModuleConfiguration = configuration;
m_directoryMembershipFingerprinterRuleSet = directoryMembershipFinterprinterRuleSet;
PathExistenceCache = new ConcurrentBigMap <AbsolutePath, PathExistence>();
FileContentManager = fileContentManager;
ServiceManager = serviceManager ?? ServiceManager.Default;
PipEnvironment = new PipEnvironment(loggingContext);
FileSystemView = fileSystemView;
m_unsafeConfiguration = unsafeConfiguration;
m_preserveOutputsSalt = preserveOutputsSalt;
LazyDeletionOfSharedOpaqueOutputsEnabled = lazyDeletionOfSharedOpaqueOutputsEnabled;
if (fileSystemView != null)
{
fileContentManager.SetLocalDiskFileSystemExistenceView(fileSystemView);
}
}
/// <summary>
/// Runs the scheduler allowing various options to be specifically set
/// </summary>
public ScheduleRunResult RunSchedulerSpecific(
PipGraph graph,
SchedulerTestHooks testHooks = null,
SchedulerState schedulerState = null,
RootFilter filter = null,
TempCleaner tempCleaner = null)
{
var config = new CommandLineConfiguration(Configuration);
// Populating the configuration may modify the configuration, so it should occur first.
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(config, Context.PathTable, bxlExeLocation: null, inTestMode: true);
BuildXLEngine.PopulateAndValidateConfiguration(config, config, Context.PathTable, LoggingContext);
FileAccessWhitelist whitelist = new FileAccessWhitelist(Context);
whitelist.Initialize(config);
IReadOnlyList <string> junctionRoots = Configuration.Engine.DirectoriesToTranslate?.Select(a => a.ToPath.ToString(Context.PathTable)).ToList();
var map = VolumeMap.TryCreateMapOfAllLocalVolumes(LoggingContext, junctionRoots);
var optionalAccessor = TryGetJournalAccessor(map);
// Although scan change journal is enabled, but if we cannot create an enabled journal accessor, then create a disabled one.
m_journalState = map == null || !optionalAccessor.IsValid
? JournalState.DisabledJournal
: JournalState.CreateEnabledJournal(map, optionalAccessor.Value);
if (config.Schedule.IncrementalScheduling)
{
// Ensure that we can scan the journal when incremental scheduling is enabled.
XAssert.IsTrue(m_journalState.IsEnabled, "Incremental scheduling requires that journal is enabled");
}
// Seal the translator if not sealed
DirectoryTranslator.Seal();
// .....................................................................................
// some dummy setup in order to get a PreserveOutputsSalt.txt file and an actual salt
// .....................................................................................
string dummyCacheDir = Path.Combine(TemporaryDirectory, "Out", "Cache");
Directory.CreateDirectory(dummyCacheDir); // EngineSchedule tries to put the PreserveOutputsSalt.txt here
ContentHash?previousOutputsSalt =
EngineSchedule.PreparePreviousOutputsSalt(LoggingContext, Context.PathTable, config);
Contract.Assert(previousOutputsSalt.HasValue);
// .....................................................................................
testHooks = testHooks ?? new SchedulerTestHooks();
Contract.Assert(!(config.Engine.CleanTempDirectories && tempCleaner == null));
using (var queue = new PipQueue(config.Schedule))
using (var testQueue = new TestPipQueue(queue, LoggingContext, initiallyPaused: false))
using (var testScheduler = new TestScheduler(
graph: graph,
pipQueue: testQueue,
context: Context,
fileContentTable: FileContentTable,
loggingContext: LoggingContext,
cache: Cache,
configuration: config,
journalState: m_journalState,
fileAccessWhitelist: whitelist,
fingerprintSalt: Configuration.Cache.CacheSalt,
directoryMembershipFingerprinterRules: new DirectoryMembershipFingerprinterRuleSet(Configuration, Context.StringTable),
tempCleaner: tempCleaner,
previousInputsSalt: previousOutputsSalt.Value,
successfulPips: null,
failedPips: null,
ipcProvider: null,
directoryTranslator: DirectoryTranslator,
testHooks: testHooks))
{
if (filter == null)
{
EngineSchedule.TryGetPipFilter(LoggingContext, Context, config, config, Expander.TryGetRootByMountName, out filter);
}
XAssert.IsTrue(testScheduler.InitForMaster(LoggingContext, filter, schedulerState), "Failed to initialized test scheduler");
testScheduler.Start(LoggingContext);
bool success = testScheduler.WhenDone().GetAwaiter().GetResult();
testScheduler.SaveFileChangeTrackerAsync(LoggingContext).Wait();
return(new ScheduleRunResult
{
Graph = graph,
Config = config,
Success = success,
PipResults = testScheduler.PipResults,
PipExecutorCounters = testScheduler.PipExecutionCounters,
PathSets = testScheduler.PathSets,
ProcessPipCountersByFilter = testScheduler.ProcessPipCountersByFilter,
ProcessPipCountersByTelemetryTag = testScheduler.ProcessPipCountersByTelemetryTag,
SchedulerState = new SchedulerState(testScheduler)
});
}
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
IKextConnection sandboxedKextConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(config != null);
LoggingContext = loggingContext;
Context = context;
// Ensure paths visible when debugging
PathTable.DebugPathTable = Context.PathTable;
Configuration = config;
PipTable = pipTable;
PathExpander = semanticPathExpander ?? SemanticPathExpander.Default;
ContentFingerprinter = new PipContentFingerprinter(
Context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null),
pathExpander: PathExpander,
pipDataLookup: pipDataLookup);
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = ipcProvider ?? IpcFactory.GetProvider();
FileContentTable = fileContentTable ?? FileContentTable.CreateNew();
Cache = pipCache;
FileAccessWhitelist = fileAccessWhitelist;
m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories;
m_sandboxedKextConnection = sandboxedKextConnection;
if (Cache == null)
{
Cache = InMemoryCacheFactory.Create(context);
}
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker);
PipGraphView = new TestPipGraphFilesystemView(Context.PathTable);
m_operationTracker = new OperationTracker(loggingContext);
var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore);
var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed;
if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled)
{
preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString());
}
State = new PipExecutionState(
config,
cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: ExecutionLogRecorder,
fileSystemView: fileSystemView,
fileContentManager: GetFileContentManager(),
directoryMembershipFinterprinterRuleSet: null,
unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: preserveOutputsSalt,
serviceManager: new DummyServiceManager());
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
