private async Task <FileContentTable> SaveAndReloadTable(FileContentTable table)
{
await SaveTable(table);
return(await LoadTable(table.EntryTimeToLive));
}
public async Task LoadOrCreateHandlesNonExistentTable()
{
FileContentTable table = await LoadOrCreateTable();
XAssert.IsNotNull(table);
}
private void ExpectHashUnknown(FileContentTable table, AbsolutePath path)
{
VersionedFileIdentityAndContentInfo?maybeKnownHash = table.TryGetKnownContentHash(path.ToString(m_pathTable));
XAssert.IsFalse(maybeKnownHash.HasValue, "Loaded table has an unexpected entry for {0}", path.ToString(m_pathTable));
}
private static Optional <CompositeGraphFingerprint> GenerateHash(
LoggingContext loggingContext,
IStartupConfiguration startUpConfig,
IConfiguration config,
PathTable pathTable,
IEvaluationFilter partialEvaluationData,
FileContentTable fileContentTable,
string commitId,
EngineTestHooksData testHooks)
{
ILayoutConfiguration layout = config.Layout;
ILoggingConfiguration logging = config.Logging;
var fingerprintTextElements = new List <(string, string)>();
using (var hasher = new CoreHashingHelper(recordFingerprintString: true))
{
CompositeGraphFingerprint fingerprint = CompositeGraphFingerprint.Zero;
AddInt(hasher, "version", GraphFingerprintVersion);
using (var qualifierHasher = new CoreHashingHelper(recordFingerprintString: false))
{
foreach (string qualifier in startUpConfig.QualifierIdentifiers.OrderBy(q => q))
{
AddText(qualifierHasher, "qualifier", qualifier);
}
fingerprint.QualifierHash = qualifierHasher.GenerateHash();
AddFingerprint(hasher, "Qualifiers", fingerprint.QualifierHash);
}
if (partialEvaluationData != null && partialEvaluationData.CanPerformPartialEvaluation)
{
using (var evaluationFilterHasher = new CoreHashingHelper(recordFingerprintString: false))
{
AddOrderedTextValues(evaluationFilterHasher, "valueName", partialEvaluationData.ValueNamesToResolveAsStrings);
AddOrderedTextValues(evaluationFilterHasher, "valuePath", partialEvaluationData.ValueDefinitionRootsToResolveAsStrings);
AddOrderedTextValues(evaluationFilterHasher, "moduleName", partialEvaluationData.ModulesToResolveAsStrings);
fingerprint.FilterHash = evaluationFilterHasher.GenerateHash();
AddFingerprint(hasher, "Values", fingerprint.FilterHash);
}
}
// These paths get embedded in the result of evaluation. So if any change we must re-evaluate
AddText(hasher, "ObjectDirectoryPath", layout.ObjectDirectory.IsValid ? layout.ObjectDirectory.ToString(pathTable) : "::null::");
AddText(hasher, "TempDirectoryPath", layout.TempDirectory.IsValid ? layout.TempDirectory.ToString(pathTable) : "::null::");
AddText(hasher, "SourceDirectoryPath", layout.SourceDirectory.IsValid ? layout.SourceDirectory.ToString(pathTable) : "::null::");
// All paths in the graph are relative to 'substTarget' (hence, 'substTarget' must be a part of the fingerprint, but 'substSource' need not be).
AddText(hasher, "substTarget", logging.SubstTarget.IsValid ? logging.SubstTarget.ToString(pathTable) : "::null::");
AddText(hasher, "IsCompressed", config.Engine.CompressGraphFiles ? "true" : "false");
AddText(hasher, "IsSkipHashSourceFile", config.Schedule.SkipHashSourceFile ? "true" : "false");
// Pip static fingerprints are not always computed because computing them slows down the graph construction by 10%-13%.
// Thus, the pip graph may and may not contain pip static fingerprints. To avoid unexpected result due to graph cache hit,
// we temporarily add the option for computing pip static fingerprints as part of our graph fingerprint until the fingerprints
// are always computed; see Task 1291638.
AddText(hasher, "ComputePipStaticFingerprints", config.Schedule.ComputePipStaticFingerprints.ToString());
if (config.Schedule.ComputePipStaticFingerprints)
{
// Pip static fingerprints are part of the graph and include the extra fingerprint salts.
// Thus, when pip static fingerprints are computed, any change to the salt will invalidate the graph because
// the pip static fingerprints will no longer be valid. Reusing the graph when the salt changes can result in
// underbuild.
var extraFingerprintSalts = new ExtraFingerprintSalts(
config,
PipFingerprintingVersion.TwoPhaseV2,
config.Cache.CacheSalt ?? string.Empty,
new Scheduler.DirectoryMembershipFingerprinterRuleSet(config, pathTable.StringTable).ComputeSearchPathToolsHash());
AddFingerprint(hasher, "ExtraFingerprintSalts", extraFingerprintSalts.CalculatedSaltsFingerprint);
}
// Config files
// Caution: Including the content hash of the config file is how changes to the default pip filter
// invalidate a cached graph. If the config file content hash is removed, the values that get
// evaluated because of it must be reflected in the values passed in to this method.
using (var configHasher = new CoreHashingHelper(recordFingerprintString: false))
{
var configFiles = new List <AbsolutePath> {
startUpConfig.ConfigFile
};
if (startUpConfig.AdditionalConfigFiles != null)
{
configFiles.AddRange(startUpConfig.AdditionalConfigFiles);
}
try
{
foreach (var configPath in configFiles
.Select(path => path.ToString(pathTable))
.OrderBy(c => c, StringComparer.OrdinalIgnoreCase))
{
AddContentHash(
configHasher,
configPath.ToUpperInvariant(),
fileContentTable.GetAndRecordContentHashAsync(configPath)
.GetAwaiter()
.GetResult()
.VersionedFileIdentityAndContentInfo.FileContentInfo.Hash);
}
fingerprint.ConfigFileHash = configHasher.GenerateHash();
AddFingerprint(hasher, "ConfigFiles", fingerprint.ConfigFileHash);
}
catch (BuildXLException ex)
{
return(LogAndReturnFailure(ex));
}
}
if (!string.IsNullOrEmpty(commitId))
{
using (var commitHasher = new CoreHashingHelper(recordFingerprintString: false))
{
commitHasher.Add("Commit", commitId);
fingerprint.BuildEngineHash = commitHasher.GenerateHash();
}
AddFingerprint(hasher, "BuildEngine", fingerprint.BuildEngineHash);
}
else
{
// BuildXL assemblies. This will invalidate the cached graph if build files change
// or if the serialization format changes.
try
{
Action <string, ContentHash> handleBuildFileAndHash = (buildFile, buildFileHash) =>
{
// Directly add to fingerprint elements for logging, but the hash is represented in the build engine hash
fingerprintTextElements.Add((buildFile, buildFileHash.ToString()));
};
var deployment = testHooks?.AppDeployment ?? AppDeployment.ReadDeploymentManifestFromRunningApp();
fingerprint.BuildEngineHash = deployment.ComputeContentHashBasedFingerprint(fileContentTable, handleBuildFileAndHash);
AddFingerprint(hasher, "BuildEngine", fingerprint.BuildEngineHash);
}
catch (BuildXLException ex)
{
Tracing.Logger.Log.FailedToComputeHashFromDeploymentManifest(loggingContext);
Tracing.Logger.Log.FailedToComputeHashFromDeploymentManifestReason(loggingContext, ex.Message);
return(default(Optional <CompositeGraphFingerprint>));
}
}
AddText(hasher, "HostOS", startUpConfig.CurrentHost.CurrentOS.ToString());
AddText(hasher, "HostCpuArchitecture", startUpConfig.CurrentHost.CpuArchitecture.ToString());
AddText(hasher, "HostIsElevated", CurrentProcess.IsElevated.ToString());
var salt = string.Empty;
if (testHooks?.GraphFingerprintSalt != null)
{
salt += testHooks.GraphFingerprintSalt.Value.ToString();
}
salt += EngineEnvironmentSettings.DebugGraphFingerprintSalt;
if (!string.IsNullOrEmpty(salt))
{
hasher.Add("GraphFingerprintSalt", salt);
}
fingerprint.OverallFingerprint = new ContentFingerprint(hasher.GenerateHash());
Tracing.Logger.Log.ElementsOfConfigurationFingerprint(
loggingContext,
fingerprint.OverallFingerprint.ToString(),
string.Join(Environment.NewLine, fingerprintTextElements.Select(kvp => "\t" + kvp.Item1 + " : " + kvp.Item2)));
return(new Optional <CompositeGraphFingerprint>(fingerprint));
}
// Local functions
void AddInt(CoreHashingHelper hasher, string key, int value)
{
hasher.Add(key, value);
fingerprintTextElements.Add(
(key, value.ToString(CultureInfo.InvariantCulture)));
}
void AddOrderedTextValues(CoreHashingHelper hasher, string key, IReadOnlyList <string> values)
{
// Limit the number of printed values to 50 to make logging more manageable. NOTE:
// values still go into fingerprint even if they are not printed.
const int maxValuesToPrint = 50;
var unprintedValueCount = values.Count - maxValuesToPrint;
int i = 0;
foreach (var value in values.OrderBy(s => s))
{
hasher.Add(key, value);
if (i < maxValuesToPrint)
{
fingerprintTextElements.Add((key, value));
}
else if (i == maxValuesToPrint)
{
fingerprintTextElements.Add((key, $"[+{unprintedValueCount} more]"));
}
i++;
}
}
void AddText(CoreHashingHelper hasher, string key, string value)
{
hasher.Add(key, value);
fingerprintTextElements.Add((key, value));
}
void AddFingerprint(CoreHashingHelper hasher, string key, Fingerprint value)
{
hasher.Add(key, value);
fingerprintTextElements.Add((key, value.ToString()));
}
void AddContentHash(CoreHashingHelper hasher, string key, ContentHash value)
{
hasher.Add(key, value);
fingerprintTextElements.Add((key, value.ToString()));
}
}
public static bool TryBuildWorkspace(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
bool topSort,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IPipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
frontEndHostController = null;
pipGraph = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = GetCommandLineConfiguration(
commandLineConfig,
configuration);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
frontEndHostController = (FrontEndHostController)controller;
// If there is an explicit engine abstraction, we set it. This is used by IDE test.
if (frontEndEngineAbstraction != null)
{
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig);
}
var config = controller.ParseConfig(mutableCommandlineConfig);
if (config == null)
{
return(false);
}
IPipGraph pipGraphBuilder = null;
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
if (frontEndEngineAbstraction == null)
{
if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(),
5000,
false);
if (topSort)
{
pipGraphBuilder = new PipGraphFragmentBuilderTopSort(engineContext, config, mountsTable.MountPathExpander);
}
else
{
pipGraphBuilder = new PipGraphFragmentBuilder(engineContext, config, mountsTable.MountPathExpander);
}
// TODO: Think more if an analyzer wants to use the real pip graph builder.
//pipGraphBuilder = new PipGraph.Builder(
// EngineSchedule.CreateEmptyPipTable(engineContext),
// engineContext,
// Scheduler.Tracing.Logger.Log,
// loggingContext,
// config,
// mountsTable.MountPathExpander,
// fingerprintSalt: config.Cache.CacheSalt,
// directoryMembershipFingerprinterRules: new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable));
if (!AddConfigurationMountsAndCompleteInitialization(config, loggingContext, mountsTable))
{
return(false);
}
IDictionary <ModuleId, MountsTable> moduleMountsTableMap;
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
}
else
{
frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config);
}
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
pipGraph = pipGraphBuilder;
//if (pipGraphBuilder != null)
//{
// pipGraph = pipGraphBuilder.Build();
// if (pipGraph == null)
// {
// // Error has been reported already.
// return false;
// }
//}
}
}
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// If workspace construction is successful, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
workspace,
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
}
return(true);
}
private static async Task <PipResult> Execute(BuildXLContext context, FileContentTable fileContentTable, IConfiguration config, Pip pip, (string substSource, string substTarget)?subst)
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
ISandboxConnection sandboxConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(config != null);
LoggingContext = loggingContext;
Context = context;
// Ensure paths visible when debugging
PathTable.DebugPathTable = Context.PathTable;
Configuration = config;
PipTable = pipTable;
PathExpander = semanticPathExpander ?? SemanticPathExpander.Default;
ContentFingerprinter = new PipContentFingerprinter(
Context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null),
pathExpander: PathExpander,
pipDataLookup: pipDataLookup);
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = ipcProvider ?? IpcFactory.GetProvider();
FileContentTable = fileContentTable ?? FileContentTable.CreateNew();
Cache = pipCache;
FileAccessWhitelist = fileAccessWhitelist;
m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories;
m_sandboxConnectionKext = sandboxConnection;
if (Cache == null)
{
Cache = InMemoryCacheFactory.Create();
}
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker);
PipGraphView = new TestPipGraphFilesystemView(Context.PathTable);
m_operationTracker = new OperationTracker(loggingContext);
var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore);
var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed;
if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled)
{
preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString());
}
State = new PipExecutionState(
config,
cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: ExecutionLogRecorder,
fileSystemView: fileSystemView,
fileContentManager: GetFileContentManager(),
directoryMembershipFinterprinterRuleSet: null,
unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: preserveOutputsSalt,
serviceManager: new DummyServiceManager());
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
private Task <FileContentTable> LoadTable(byte entryTimeToLive = FileContentTable.DefaultTimeToLive)
{
return(FileContentTable.LoadAsync(LoggingContext, GetFullPath(Table), entryTimeToLive));
}
/// <summary>
/// Creates a file content table.
/// </summary>
protected FileContentTable CreateFileContentTable()
{
Contract.Ensures(Contract.Result <FileContentTable>() != null);
return(FileContentTable.CreateNew(LoggingContext));
}
private Task <FileContentTable> LoadOrCreateTable(ushort entryTimeToLive = FileContentTable.DefaultTimeToLive)
{
return(FileContentTable.LoadOrCreateAsync(LoggingContext, GetFullPath(Table), entryTimeToLive: entryTimeToLive));
}
private async Task ProcessWindowsCallHelper(
string functionName,
SandboxConfiguration config = null,
IEnumerable <string> extraDependencies = null,
IEnumerable <string> extraOutputs = null,
int callCount = 1,
string commandPrefix = "",
bool readsAndWritesDirectories = false,
bool untrackedOutputs = false,
string[] expectedWarningStrings = null,
string[] expectedErrorStrings = null)
{
if (config == null)
{
config = new SandboxConfiguration {
FileAccessIgnoreCodeCoverage = true, FailUnexpectedFileAccesses = true
};
}
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var fileContentTable = FileContentTable.CreateNew();
// have to force the config for truncation
config.OutputReportingMode = OutputReportingMode.FullOutputOnWarningOrError;
bool expectSuccess = expectedErrorStrings == null && expectedWarningStrings == null;
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
{
string currentCodeFolder = Path.GetDirectoryName(AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly()));
Contract.Assume(currentCodeFolder != null);
string executable = Path.Combine(currentCodeFolder, DetourTestFolder, "DetoursTests.exe");
string workingDirectory = tempFiles.GetUniqueDirectory();
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
XAssert.IsTrue(File.Exists(executable), "Could not find the test file: " + executable);
FileArtifact executableFileArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
var extraUntrackedScopes = new List <AbsolutePath>();
var dependencies = new List <FileArtifact> {
executableFileArtifact
};
if (extraDependencies != null)
{
foreach (string file in extraDependencies)
{
string filePath = Path.Combine(workingDirectory, file);
AbsolutePath path = AbsolutePath.Create(pathTable, filePath);
if (readsAndWritesDirectories)
{
Directory.CreateDirectory(filePath);
// We don't support directories as inputs in BuildXL yet.
extraUntrackedScopes.Add(path);
}
else
{
File.WriteAllText(filePath, "Definitely a file");
FileArtifact fileArtifact = FileArtifact.CreateSourceFile(path);
dependencies.Add(fileArtifact);
}
}
}
var outputs = new List <FileArtifactWithAttributes>();
if (extraOutputs != null)
{
foreach (string file in extraOutputs)
{
string filePath = Path.Combine(workingDirectory, file);
AbsolutePath path = AbsolutePath.Create(pathTable, filePath);
if (readsAndWritesDirectories)
{
// We don't support directory outputs in BuildXL at the moment, so e.g. deleting a directory needs to be untracked.
extraUntrackedScopes.Add(path);
}
else if (untrackedOutputs)
{
extraUntrackedScopes.Add(path);
}
else
{
FileArtifact fileArtifact = FileArtifact.CreateSourceFile(path).CreateNextWrittenVersion();
outputs.Add(fileArtifact.WithAttributes());
}
}
}
var tempDirectory = tempFiles.GetUniqueDirectory();
var environmentVariables = new List <EnvironmentVariable>();
var environmentValue = new PipDataBuilder(pathTable.StringTable);
var tempPath = AbsolutePath.Create(pathTable, tempDirectory);
environmentValue.Add(tempPath);
environmentVariables.Add(new EnvironmentVariable(StringId.Create(pathTable.StringTable, "TMP"), environmentValue.ToPipData(" ", PipDataFragmentEscaping.NoEscaping)));
environmentVariables.Add(new EnvironmentVariable(StringId.Create(pathTable.StringTable, "TEMP"), environmentValue.ToPipData(" ", PipDataFragmentEscaping.NoEscaping)));
var untrackedPaths = CmdHelper.GetCmdDependencies(pathTable);
var untrackedScopes = extraUntrackedScopes.Concat(CmdHelper.GetCmdDependencyScopes(pathTable).Concat(new[] { tempPath })).Distinct();
var pip = new Process(
executableFileArtifact,
workingDirectoryAbsolutePath,
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.NoEscaping, commandPrefix + functionName + "Logging"),
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .From(environmentVariables),
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pathTable),
null,
null,
ReadOnlyArray <FileArtifact> .From(dependencies),
ReadOnlyArray <FileArtifactWithAttributes> .From(outputs),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(untrackedPaths),
ReadOnlyArray <AbsolutePath> .From(untrackedScopes),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
if (expectSuccess)
{
await AssertProcessSucceeds(
context,
config,
pip);
}
else
{
await AssertProcessCompletesWithStatus(
SandboxedProcessPipExecutionStatus.ExecutionFailed,
context,
config,
pip,
null);
}
}
int expectedErrorCount = 0;
int expectedWarningCount = 0;
IEnumerable <string> requiredLogMessageSubstrings = new string[] { };
if (expectedErrorStrings != null)
{
expectedErrorCount = expectedErrorStrings.Count();
requiredLogMessageSubstrings = requiredLogMessageSubstrings.Concat(expectedErrorStrings);
}
if (expectedWarningStrings != null)
{
expectedWarningCount = expectedWarningStrings.Count();
requiredLogMessageSubstrings = requiredLogMessageSubstrings.Concat(expectedWarningStrings);
}
SetExpectedFailures(expectedErrorCount, expectedWarningCount, requiredLogMessageSubstrings.ToArray());
}
public async Task ParallelRecordForHardlinksRemembersHash()
{
if (!FileUtilities.IsPreciseFileVersionSupportedByEnlistmentVolume)
{
// TODO: Currently fails on OS that does not support precise file version.
return;
}
var table = FileContentTable.CreateNew(LoggingContext);
const int ThreadCount = 16;
const int IterationCount = 20;
const string OriginalFile = "Original";
WriteFile(OriginalFile, "Data");
CreateHardLinkStatus testLinkStatus = FileUtilities.TryCreateHardLink(GetFullPath("TestLink"), GetFullPath(OriginalFile));
if (testLinkStatus == CreateHardLinkStatus.FailedSinceNotSupportedByFilesystem)
{
return;
}
XAssert.AreEqual(CreateHardLinkStatus.Success, testLinkStatus);
var linkTasks = new Task[ThreadCount];
var barrier = new Barrier(ThreadCount);
for (int j = 0; j < IterationCount; j++)
{
for (int i = 0; i < ThreadCount; i++)
{
int threadId = i;
linkTasks[threadId] = Task.Run(
() =>
{
string relativePath = "Link-" + threadId;
AbsolutePath path = GetFullPath(m_pathTable, relativePath);
barrier.SignalAndWait();
FileUtilities.DeleteFile(GetFullPath(relativePath));
CreateHardLinkStatus linkStatus = FileUtilities.TryCreateHardLink(GetFullPath(relativePath), GetFullPath(OriginalFile));
#if PLATFORM_OSX
// Catalina seems to have issues when doing mutli-threaded delete / link operations, let's retry several times..
if (linkStatus != CreateHardLinkStatus.Success)
{
var count = 0;
while (count < ThreadCount)
{
FileUtilities.DeleteFile(GetFullPath(relativePath));
linkStatus = FileUtilities.TryCreateHardLink(GetFullPath(relativePath), GetFullPath(OriginalFile));
count++;
}
}
#endif
XAssert.AreEqual(CreateHardLinkStatus.Success, linkStatus);
RecordContentHash(table, path, s_hashA);
});
}
await Task.WhenAll(linkTasks);
ExpectHashKnown(table, GetFullPath(m_pathTable, OriginalFile), s_hashA);
}
}
private static bool TryBuildWorkspaceInternal(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IMutablePipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
bool forIDE,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
frontEndHostController = null;
pipGraph = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = GetCommandLineConfiguration(
commandLineConfig,
configuration);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
frontEndHostController = (FrontEndHostController)controller;
// If there is an explicit engine abstraction, we set it. This is used by the IDE.
if (frontEndEngineAbstraction != null)
{
// The IDE engine typically doesn't have mounts configured. We do it here if they haven't been configured yet.
// Observe these are just the default mounts used for config evaluation.
if (frontEndEngineAbstraction is BasicFrontEndEngineAbstraction basicEngine && !frontEndEngineAbstraction.GetMountNames("Script", BuildXL.Utilities.ModuleId.Invalid).Any())
{
// If this fails we just ignore the failure. Mounts not being properly configured doesn't prevent the IDE plugin from working.
basicEngine.TryPopulateWithDefaultMountsTable(loggingContext, engineContext, mutableCommandlineConfig, mutableCommandlineConfig.Startup.Properties);
}
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig);
}
else
{
// Otherwise we construct one with all mounts populated for config evaluation
var configurationEngine = new BasicFrontEndEngineAbstraction(engineContext.PathTable, engineContext.FileSystem, mutableCommandlineConfig);
if (!configurationEngine.TryPopulateWithDefaultMountsTable(loggingContext, engineContext, mutableCommandlineConfig, mutableCommandlineConfig.Startup.Properties))
{
// Errors are logged already
return(false);
}
frontEndEngineAbstraction = configurationEngine;
}
var config = controller.ParseConfig(frontEndEngineAbstraction, mutableCommandlineConfig);
if (config == null)
{
return(false);
}
IMutablePipGraph pipGraphBuilder = null;
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
// For the IDE case, we want to make sure all config-specific mounts are properly populated
if (forIDE && frontEndEngineAbstraction is BasicFrontEndEngineAbstraction languageServiceEngine)
{
Contract.AssertNotNull(frontEndEngineAbstraction);
AddConfigurationMounts(config, mountsTable);
languageServiceEngine.SetMountsTable(mountsTable);
}
if (frontEndEngineAbstraction == null)
{
if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule))
{
frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(loggingContext),
5000,
false,
controller.RegisteredFrontEnds);
var searchPathToolsHash = new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable).ComputeSearchPathToolsHash();
pipGraphBuilder = new PipGraph.Builder(
EngineSchedule.CreateEmptyPipTable(engineContext),
engineContext,
Pips.Tracing.Logger.Log,
loggingContext,
config,
mountsTable.MountPathExpander,
fingerprintSalt: config.Cache.CacheSalt,
searchPathToolsHash: searchPathToolsHash);
// Observe mount table is completed during workspace construction
AddConfigurationMounts(config, mountsTable);
IDictionary <ModuleId, MountsTable> moduleMountsTableMap;
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
}
else
{
frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config);
}
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
pipGraph = pipGraphBuilder;
}
}
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// If workspace construction is successful, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
workspace,
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
}
return(true);
}
/// <summary>
/// Creates a file content table.
/// </summary>
protected FileContentTable CreateFileContentTable()
{
return(FileContentTable.CreateNew(LoggingContext));
}
private Task SaveTable(FileContentTable table)
{
return(table.SaveAsync(GetFullPath(Table)));
}
private static bool TryBuildPipGraphFragment(
ICommandLineConfiguration commandLineConfig,
PipGraphFragmentGeneratorConfiguration pipGraphFragmentGeneratorConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter)
{
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = CompleteCommandLineConfiguration(commandLineConfig);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics();
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
FrontEndHostController frontEndHostController = (FrontEndHostController)controller;
var config = controller.ParseConfig(mutableCommandlineConfig);
if (config == null)
{
return(false);
}
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
new EmptyTwoPhaseFingerprintStore())))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
FrontEndEngineAbstraction frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(loggingContext),
5000,
false,
controller.RegisteredFrontEnds);
var pipGraphBuilder = pipGraphFragmentGeneratorConfig.TopSort
? new PipGraphFragmentBuilderTopSort(engineContext, config, mountsTable.MountPathExpander)
: new PipGraphFragmentBuilder(engineContext, config, mountsTable.MountPathExpander);
// Observe mount table is completed during workspace construction
AddConfigurationMounts(config, mountsTable);
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out var moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
// Error should have been reported already
return(false);
}
if (!SerializeFragmentIfRequested(pipGraphFragmentGeneratorConfig, frontEndContext, pipGraphBuilder))
{
// Error should have been reported already
return(false);
}
}
}
return(true);
}
private Task <FileContentTable> LoadOrCreateTable(byte entryTimeToLive = FileContentTable.DefaultTimeToLive)
{
return(FileContentTable.LoadOrCreateAsync(GetFullPath(Table), entryTimeToLive: entryTimeToLive));
}
private static async Task <bool> CreateAndRunPip(
PipProgram program,
string tempDirectory,
string outFile,
IEnumerable <string> restInstructions,
bool is64Bit)
{
Contract.Requires(restInstructions != null);
Contract.Requires(tempDirectory != null);
Contract.Requires(!string.IsNullOrEmpty(outFile));
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
using (var fileAccessListener = new FileAccessListener(Events.Log))
{
fileAccessListener.RegisterEventSource(BuildXL.Processes.ETWLogger.Log);
var loggingContext = BuildXLTestBase.CreateLoggingContextForTest();
var fileContentTable = FileContentTable.CreateNew(loggingContext);
var config = ConfigurationHelpers.GetDefaultForTesting(context.PathTable, AbsolutePath.Create(context.PathTable, Path.Combine(tempDirectory, "config.dc")));
config.Sandbox.LogObservedFileAccesses = true;
Pip pip = null;
var instructions = restInstructions as string[] ?? restInstructions.ToArray();
switch (program)
{
case PipProgram.Cmd:
pip = CreateCmdPip(context, tempDirectory, outFile, is64Bit);
break;
case PipProgram.Self:
pip = CreateSelfPip(context, tempDirectory, outFile, instructions, is64Bit);
break;
}
Contract.Assume(pip != null);
var isSubstUsed = FileUtilities.TryGetSubstSourceAndTarget(tempDirectory, out var substSource, out var substTarget, out var errorMessage);
XAssert.IsFalse(!isSubstUsed && errorMessage != null, errorMessage);
PipResult executeResult = await Execute(
context,
fileContentTable,
config,
pip,
isSubstUsed
?(substSource, substTarget)
: default((string, string)?));
bool valid = false;
switch (program)
{
case PipProgram.Cmd:
valid = ValidateCmd(fileAccessListener.FileAccesses, outFile, is64Bit);
break;
case PipProgram.Self:
valid = ValidateSelf(
fileAccessListener.FileAccesses,
instructions.Length > 0 ? instructions[0] : string.Empty,
outFile,
is64Bit);
break;
}
return(executeResult.Status == PipResultStatus.Succeeded && valid);
}
}
