public void JSResolverForcesReparsePointResolution(bool?enableFullReparsePointResolving, bool expectDFA)
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
{
var tempDirectory = tempFiles.GetDirectory("source");
// Create a symlink source -> target
var source = Path.Combine(tempDirectory, "source").Replace("\\", "/");
var target = Path.Combine(tempDirectory, "target").Replace("\\", "/");
FileUtilities.CreateDirectory(target);
XAssert.PossiblySucceeded(FileUtilities.TryCreateReparsePoint(source, target, ReparsePointType.Junction));
// Run a project that writes a file under 'source'. Only specify 'source' as an output dir (and not 'target').
var config = (CommandLineConfiguration)Build(additionalOutputDirectories: $"[p`{source}`]", enableFullReparsePointResolving: enableFullReparsePointResolving)
.AddJavaScriptProject(
"@ms/project-A",
"src/A",
$@"
var fs = require('fs');
fs.writeFileSync('{source}/out.txt', 'hello');")
.PersistSpecsAndGetConfiguration();
config.Engine.UnsafeAllowOutOfMountWrites = true;
((UnsafeSandboxConfiguration)(config.Sandbox.UnsafeSandboxConfiguration)).UnexpectedFileAccessesAreErrors = false;
var engineResult = RunYarnProjects(config);
// If reparse point resolution is enabled we should get a DFA because 'target' is not a directory where outputs are expected
Assert.True(engineResult.IsSuccess);
if (expectDFA)
{
AssertWarningEventLogged(global::BuildXL.Scheduler.Tracing.LogEventId.FileMonitoringWarning);
AssertWarningEventLogged(global::BuildXL.Scheduler.Tracing.LogEventId.ProcessNotStoredToCacheDueToFileMonitoringViolations, 2);
}
}
}
public void AllSpecsAreSerialized()
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
var config = Build()
.EmptyConfiguration()
.AddSpec("spec1.dsc", "export const x = 42;")
.AddSpec("spec2.dsc", "export const y = 53;")
.AddSpec("spec3.dsc", "export const z = 64;")
.RootSpec("spec1.dsc")
.PersistSpecsAndGetConfiguration(enableSpecCache: true);
var specs = RunAndRetrieveSpecs(config, appDeployment);
// Now that the controller is disposed, all public facades + asts have been serialized
// We create a new public facade provider to verify everything was saved. A basic front end engine abstraction should be fine, since
// the retrieve method only relies on the engine abstraction to compute the proper hashes.
var publicFacadeProvider = GetPublicFacadeProvider(config);
foreach (var spec in specs)
{
var result = publicFacadeProvider.TryGetPublicFacadeWithAstAsync(spec);
Assert.True(result != null);
}
}
}
public void EnvVarExpansionShouldInvalidateTheCacheOnChange(bool envVarIsSet)
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
var testRoot = tempFiles.GetUniqueDirectory(PathTable).ToString(PathTable);
// The env var is explicit set (or explicitly deleted)
Environment.SetEnvironmentVariable("MyEnvVar", envVarIsSet? "MyValue" : null);
string spec = @"export const s = Environment.expandEnvironmentVariablesInString('%MyEnvVar%');";
var buildDefinition = CreateDefinition(spec);
// The first run should be a miss
RunAndAssertGraphCacheMiss(WriteSpecs(testRoot, buildDefinition), appDeployment);
// A second run should be a hit
RunAndAssertGraphCacheHit(WriteSpecs(testRoot, buildDefinition), appDeployment);
// Change the environment variable the expanded string referenced. We should now
// get a cache miss even when the env var was not set to begin with
Environment.SetEnvironmentVariable("MyEnvVar", "MyOTHERValue");
RunAndAssertGraphCacheMiss(WriteSpecs(testRoot, buildDefinition), appDeployment);
}
}
protected BuildXLEngineResult RunYarnProjects(
ICommandLineConfiguration config,
TestCache testCache = null,
IDetoursEventListener detoursListener = null)
{
// This bootstraps the 'repo'
if (!YarnInit(config))
{
throw new InvalidOperationException("Yarn init failed.");
}
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
((CommandLineConfiguration)config).Engine.Phase = Phase;
((CommandLineConfiguration)config).Sandbox.FileSystemMode = FileSystemMode.RealAndMinimalPipGraph;
var engineResult = CreateAndRunEngine(
config,
appDeployment,
testRootDirectory: null,
rememberAllChangedTrackedInputs: true,
engine: out var engine,
testCache: testCache,
detoursListener: detoursListener);
return(engineResult);
}
}
public async Task ErrorHandling()
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var storage = (ISandboxedProcessFileStorage)tempFiles;
var fileName = storage.GetFileName(SandboxedProcessFile.StandardOutput);
SandboxedProcessOutput output;
using (var fileStream = FileUtilities.CreateFileStream(fileName, FileMode.CreateNew, FileAccess.Write, FileShare.None, allowExcludeFileShareDelete: true))
{
var content = new string('S', 100);
var outputBuilder =
new SandboxedProcessOutputBuilder(Encoding.UTF8, content.Length / 2, tempFiles, SandboxedProcessFile.StandardOutput, null);
// NOTE: this only holds on Windows
// The specified content plus a NewLine will exceed the max memory length.
// Thus, the output builder will try to write the content to a file.
// However, the file is not writable (this runs in a using clause that already opened the file).
// Thus, this will internally fail, but not yet throw an exception.
outputBuilder.AppendLine(content);
output = outputBuilder.Freeze();
XAssert.IsTrue(output.HasException);
}
await Assert.ThrowsAsync <BuildXLException>(() => output.ReadValueAsync());
await Assert.ThrowsAsync <BuildXLException>(() => output.SaveAsync());
Assert.Throws <BuildXLException>(() => output.CreateReader());
}
}
public async Task TombstoneFileDoesNotRepresentARequiredOutput()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName();
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var renamedFile = $"{outputFile}.renamed";
var renamedFilePath = AbsolutePath.Create(pathTable, renamedFile);
// Arguments to create an output file that gets immediately renamed.
// However, both files are marked as required, even though the original one
// is not there after the rename happens
var arguments = $"echo hi > {outputFile} && move {outputFile} {renamedFile}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(renamedFilePath), FileExistence.Required)
});
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
// the move command under the console seems to have some issue with the detours policy
// This is orthogonal to the test, and we don't care about detours at this point
var pipExecutionResult = await RunProcess(context, pip, failUnexpectedFileAccesses : false);
// The redirected output is created as a tombstone file, but the sandboxed pip executor should report it as an absent file
AssertErrorEventLogged(ProcessesLogEventId.PipProcessMissingExpectedOutputOnCleanExit);
AssertErrorEventLogged(ProcessesLogEventId.PipProcessExpectedMissingOutputs);
}
}
public void TestFileAndDirectoryExistence()
{
// This test makes sure that real input tracker works fine with file/directory existence logic
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
var testRoot = tempFiles.GetUniqueDirectory(PathTable).ToString(PathTable);
string spec = @"
const fileShouldExist = File.exists(f`foo/a.txt`);
export const r1 = (()=>{Contract.assert(fileShouldExist === true);})();
const fileShouldNotExist = File.exists(f`foo/missing.txt`);
export const r2 = (()=>{Contract.assert(fileShouldNotExist === false);})();
const directoryShouldExist = Directory.exists(d`foo`);
export const r3 = (()=>{Contract.assert(directoryShouldExist === true);})();
const directoryShouldNotExist = Directory.exists(d`bar`);
export const r4 = (()=>{Contract.assert(directoryShouldNotExist === false);})();
";
var config = Build()
.EmptyConfiguration()
.TestRootDirectory(testRoot)
.AddSpec("spec1.dsc", spec)
.AddFile("foo/a.txt", "")
.PersistSpecsAndGetConfiguration();
var specs = RunAndRetrieveSpecs(config, appDeployment);
}
}
public void TestGraphSerializationWithLightEdges()
{
using (var tempStorage = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
MutableDirectedGraph graph;
NodeId[] nodes;
CreateGraphWithLightEdges(out graph, out nodes);
string fileName = tempStorage.GetUniqueFileName();
using (FileStream fileStream = File.Open(fileName, FileMode.Create))
{
var writer = new BuildXLWriter(debug: false, stream: fileStream, leaveOpen: false, logStats: false);
graph.Serialize(writer);
}
MutableDirectedGraph newGraph;
using (FileStream fileStream = File.Open(fileName, FileMode.Open))
{
var reader = new BuildXLReader(debug: false, stream: fileStream, leaveOpen: false);
newGraph = MutableDirectedGraph.Deserialize(reader);
}
XAssert.AreEqual(newGraph.NodeCount, nodes.Length);
XAssert.AreEqual(newGraph.EdgeCount, 5);
XAssert.IsTrue(newGraph.ContainsEdge(nodes[0], nodes[1], isLight: true));
XAssert.IsTrue(newGraph.ContainsEdge(nodes[0], nodes[1], isLight: false));
XAssert.IsFalse(newGraph.ContainsEdge(nodes[1], nodes[0], isLight: true));
XAssert.IsFalse(newGraph.ContainsEdge(nodes[1], nodes[0], isLight: false));
XAssert.IsFalse(newGraph.ContainsEdge(nodes[1], nodes[3], isLight: true));
}
}
public async Task ProcessInContainerGeneratingNestedOutputsPassesAllSandboxValidations()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName(@"outputs\");
var outputFileNested = tempFiles.GetUniqueFileName(@"outputs\nested\");
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var outputFileNestedPath = AbsolutePath.Create(pathTable, outputFileNested);
FileUtilities.CreateDirectory(outputFileNestedPath.GetParent(pathTable).ToString(pathTable));
// Arguments to create two output files, one nested under the other
var arguments = $"echo hi > {outputFile} && echo bye > {outputFileNested}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFileNestedPath), FileExistence.Required)
});
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
var pipExecutionResult = await RunProcess(context, pip);
// Observe that the fact the execution result succeeds means that the expected outputs are in their expected place
XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
}
}
public void QualifierBasedTagsAreSet()
{
// Writes a file using a process pip with user defined tags 'test1' and 'test2'
string spec = @"
import {Cmd, Transformer} from 'Sdk.Transformers';
export declare const qualifier: {platform: 'x86' | 'x64'; configuration: 'debug' | 'release'};
const outDir = Context.getNewOutputDirectory('tags');
const result = Transformer.execute({
tool: {" +
$"exe: f`{(OperatingSystemHelper.IsUnixOS ? "/bin/sh" : @"${Environment.getPathValue(""COMSPEC"")}")}`"
+ @"},
workingDirectory: d`.`,
arguments: [ Cmd.rawArgument('" + $"{(OperatingSystemHelper.IsUnixOS ? "-c echo test > obj/a.txt" : @"/d /c echo test > obj\a.txt")}" + @"'),
],
outputs: [
p`${outDir}/obj/a.txt`,
],
tags: ['test1', 'test2']
});
";
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// Configure the initial qualifier to be configuration=debug, platform=x64
var config = Build()
.Configuration(@"
config({
qualifiers: {
defaultQualifier: {
configuration: 'debug',
platform: 'x64'
}
}
});")
.AddSpec(spec)
.PersistSpecsAndGetConfiguration();
// Scheduling is enough, we just want to inspect the tags
((CommandLineConfiguration)config).Engine.Phase = global::BuildXL.Utilities.Configuration.EnginePhases.Schedule;
var engineResult = CreateAndRunEngine(
config,
appDeployment,
testRootDirectory: null,
rememberAllChangedTrackedInputs: false,
engine: out var engine);
XAssert.IsTrue(engineResult.IsSuccess);
// The corresponding process pip should have both the user defined tags and the current qualifier tags
var process = (PipsOperations.Process)engineResult.EngineState.PipGraph.RetrievePipsOfType(PipsOperations.PipType.Process).Single();
XAssert.Contains(process.Tags.Select(tag => tag.ToString(StringTable)), "test1", "test2", "platform=x64", "configuration=debug");
}
}
public async Task CorrelateMoveFileAsync()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
{
AbsolutePath sourceDirectory = tempFiles.GetDirectory(pathTable, "Source");
AbsolutePath sourceFile = sourceDirectory.Combine(pathTable, "SourceFile.txt");
var destinationFile = tempFiles.GetFileName(pathTable, "DestinationFile.txt");
WriteFile(pathTable, sourceFile, "content");
var process = CreateDetourProcess(
context,
pathTable,
tempFiles,
argumentStr: "CorrelateMoveFile",
inputFiles: ReadOnlyArray <FileArtifact> .Empty,
inputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
outputFiles: ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
FileArtifactWithAttributes.Create(
FileArtifact.CreateOutputFile(destinationFile), FileExistence.Required)),
outputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(sourceDirectory));
var correlator = new Correlator(pathTable);
SandboxedProcessPipExecutionResult result = await RunProcessAsync(
pathTable : pathTable,
ignoreSetFileInformationByHandle : false,
ignoreZwRenameFileInformation : false,
monitorNtCreate : true,
ignoreReparsePoints : false,
ignoreNonCreateFileReparsePoints : false,
monitorZwCreateOpenQueryFile : false,
context : context,
pip : process,
detoursListener : correlator,
errorString : out _);
VerifyNormalSuccess(context, result);
XAssert.IsTrue(File.Exists(destinationFile.ToString(pathTable)));
var toVerify = new List <(AbsolutePath, RequestedAccess, FileAccessStatus)>
{
(sourceFile, RequestedAccess.ReadWrite, FileAccessStatus.Allowed),
(destinationFile, RequestedAccess.Write, FileAccessStatus.Allowed)
};
VerifyFileAccesses(context, result.AllReportedFileAccesses, toVerify.ToArray());
correlator.VerifyCorrelation(new Correlator.VerifiedCorrelation(
destinationFile.ToString(pathTable),
ReportedFileOperation.MoveFileWithProgressDest,
sourceFile.ToString(pathTable),
ReportedFileOperation.MoveFileWithProgressSource));
}
}
protected static AppDeployment CreateAppDeployment(TempFileStorage tempFiles)
{
string manifestPath = tempFiles.GetFileName(AppDeployment.DeploymentManifestFileName);
File.WriteAllText(
Path.Combine(Path.GetDirectoryName(manifestPath), AppDeployment.DeploymentManifestFileName),
AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly()));
AppDeployment appDeployment = AppDeployment.ReadDeploymentManifest(Path.GetDirectoryName(manifestPath), AppDeployment.DeploymentManifestFileName, skipManifestCheckTestHook: true);
return appDeployment;
}
private static Process CreateOutputFileProcess(BuildXLContext context, TempFileStorage tempFiles, AbsolutePath outputFilePath)
{
// Arguments to create an output file
var arguments = $"echo hi > {outputFilePath.ToString(context.PathTable)}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[] { FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required) });
var pip = CreateConsoleProcessInContainer(context, tempFiles, context.PathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
return(pip);
}
public async Task IsolationLevelControlsWriteRedirection(ContainerIsolationLevel containerIsolationLevel)
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName(@"fileOutputs\");
var opaqueOutputFile = tempFiles.GetUniqueFileName(@"directoryOutputs\");
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var opaqueOutputFilePath = AbsolutePath.Create(pathTable, opaqueOutputFile);
var arguments = $"echo hi > {outputFile} && echo bye > {opaqueOutputFile}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
});
var opaqueOutputs = ReadOnlyArray <DirectoryArtifact> .FromWithoutCopy(
new[]
{
new DirectoryArtifact(opaqueOutputFilePath.GetParent(pathTable), 1, isSharedOpaque: containerIsolationLevel == ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories),
}
);
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, opaqueOutputs, containerIsolationLevel);
var pipExecutionResult = await RunProcess(context, pip);
XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
var redirectedDirForFiles = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "fileOutputs");
var redirectedFile = redirectedDirForFiles.Combine(pathTable, outputFilePath.GetName(pathTable)).ToString(pathTable);
var redirectedDirForDirectories = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "directoryOutputs");
var redirectedOpaqueFile = redirectedDirForDirectories.Combine(pathTable, opaqueOutputFilePath.GetName(pathTable)).ToString(pathTable);
// Make sure outputs got redirected based on the configured isolation level
switch (containerIsolationLevel)
{
case ContainerIsolationLevel.IsolateOutputFiles:
XAssert.IsTrue(File.Exists(redirectedFile));
break;
case ContainerIsolationLevel.IsolateExclusiveOpaqueOutputDirectories:
case ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories:
XAssert.IsTrue(File.Exists(redirectedOpaqueFile));
break;
}
}
}
public void PublicFacadeAndAstIsReusedForCleanSpecs()
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// The same testRootDirectory is used for both invocations, so the spec cache can be reused.
var testRoot = Path.Combine(TestOutputDirectory, Guid.NewGuid().ToString("N"));
var config = BuildWithoutDefautlLibraries()
.AddPrelude()
.EmptyConfiguration()
.TestRootDirectory(testRoot)
.AddSpec("spec1.dsc", "export const x = 42;")
.AddSpec("spec2.dsc", "export const y = x;")
.AddSpec("spec3.dsc", "export const w = y;")
.AddSpec("spec4.dsc", "export const z = w;")
.PersistSpecsAndGetConfiguration(enableSpecCache: true);
RunAndRetrieveSpecs(config, appDeployment);
// Now we change spec3 in a way the binding fingerprint does not change for it
config = BuildWithoutDefautlLibraries()
.AddPrelude()
.EmptyConfiguration()
.TestRootDirectory(testRoot)
.AddSpec("spec3.dsc", "export const w = y;//safe change")
.PersistSpecsAndGetConfiguration(cleanExistingDirectory: false, enableSpecCache: true);
// Since the spec2spec binding should be spec1 <- spec2 <- spec3 <-spec 4, by changing spec3 without changing the binding information, spec3 and 4
// should become dirty. On the other hand, spec1 and spec2 should be reused from the previous run
using (var controller = RunEngineAndGetFrontEndHostController(config, appDeployment, testRoot, rememberAllChangedTrackedInputs: true))
{
var workspace = controller.Workspace;
var sourceFiles = workspace.GetAllSourceFiles();
foreach (var sourceFile in sourceFiles)
{
var fileName = Path.GetFileName(sourceFile.Path.AbsolutePath);
if (fileName.Equals("spec1.dsc", StringComparison.OrdinalIgnoreCase) ||
fileName.Equals("spec2.dsc", StringComparison.OrdinalIgnoreCase))
{
Assert.True(sourceFile.IsPublicFacade);
}
else if (fileName.Equals("spec3.dsc", StringComparison.OrdinalIgnoreCase) ||
fileName.Equals("spec4.dsc", StringComparison.OrdinalIgnoreCase))
{
Assert.False(sourceFile.IsPublicFacade, $"{sourceFile.Path.AbsolutePath} should not be a public facade.");
}
}
}
}
}
public void PublicFacadeAndAstIsReusedForSpecsWithoutPublicSurfaceChanges()
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// The same testRootDirectory is used for both invocations, so the spec cache can be reused.
var testRoot = tempFiles.GetUniqueDirectory(PathTable).ToString(PathTable);
var config = BuildWithoutDefautlLibraries()
.AddPrelude()
.EmptyConfiguration()
.TestRootDirectory(testRoot)
.AddSpec("spec1.dsc", "export const x = 42;")
.AddSpec("spec2.dsc", "export const y = x;")
.AddSpec("spec3.dsc", "export const w = y;")
.AddSpec("spec4.dsc", "export const z = w;")
.AddSpec("spec5.dsc", "export const notUsedYet = 42;")
.PersistSpecsAndGetConfiguration(enableSpecCache: true);
RunAndRetrieveSpecs(config, appDeployment);
// Now we change spec3 in a way the binding fingerprint does not change for it
config = BuildWithoutDefautlLibraries()
.AddPrelude()
.EmptyConfiguration()
.TestRootDirectory(testRoot)
.AddSpec("spec3.dsc", "export const w = notUsedYet;")
.PersistSpecsAndGetConfiguration(cleanExistingDirectory: false, enableSpecCache: true);
// Even though the spec3 has changed reasonably and now has another dependency
// it should not prevent us from using public facades for unafected specs.
using (var controller = RunEngineAndGetFrontEndHostController(config, appDeployment, testRoot, rememberAllChangedTrackedInputs: true))
{
var workspace = controller.Workspace;
var sourceFiles = workspace.GetAllSourceFiles();
foreach (var sourceFile in sourceFiles)
{
if (Path.GetFileName(sourceFile.Path.AbsolutePath).EndsWith("spec1.dsc") ||
Path.GetFileName(sourceFile.Path.AbsolutePath).EndsWith("spec2.dsc") ||
Path.GetFileName(sourceFile.Path.AbsolutePath).EndsWith("spec5.dsc"))
{
Assert.True(sourceFile.IsPublicFacade, $"{sourceFile.Path.AbsolutePath} should be a public facade.");
}
}
}
}
}
public async Task ProcessInContainerPassesAllSandboxValidations()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName();
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var pip = CreateOutputFileProcess(context, tempFiles, outputFilePath);
var pipExecutionResult = await RunProcess(context, pip);
// Observe that the fact the execution result succeeds means that the expected outputs are in their expected place
XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
}
}
public void PerturbingFileChangeTrackerShouldNotAffectMatchingInput()
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
var testRoot = tempFiles.GetUniqueDirectory(PathTable).ToString(PathTable);
const string SpecFile = "spec.dsc";
var buildDefinition = CreateDefinition(@"export const x = 42;", SpecFile);
var configuration = WriteSpecs(testRoot, buildDefinition);
RunAndAssertGraphCacheMiss(configuration, appDeployment, rememberAllChangedTrackedInputs: true);
// Copy tracker.
var trackerPath = Path.Combine(
configuration.Layout.EngineCacheDirectory.ToString(FrontEndContext.PathTable),
EngineSerializer.PreviousInputsJournalCheckpointFile);
var copiedTrackerPath = trackerPath + ".copy";
XAssert.IsTrue(File.Exists(trackerPath));
File.Copy(trackerPath, copiedTrackerPath);
// Modify spec.
buildDefinition.AppendToSpecContent(SpecFile, " ");
RunAndAssertGraphCacheMiss(WriteSpecs(testRoot, buildDefinition), appDeployment, rememberAllChangedTrackedInputs: true);
// Perturb tracker, by moving the old one to its original location, and thus will have wrong envelope id.
File.Delete(trackerPath);
File.Move(copiedTrackerPath, trackerPath);
// Modify spec again.
buildDefinition.AppendToSpecContent(SpecFile, " ");
var hostController = RunEngineAndGetFrontEndHostController(
WriteSpecs(testRoot, buildDefinition),
appDeployment,
null,
true,
engineTestHooksData => {
XAssert.IsTrue(ContainsFileName(engineTestHooksData.GraphReuseResult.InputChanges.ChangedPaths, SpecFile));
XAssert.IsFalse(ContainsFileName(engineTestHooksData.GraphReuseResult.InputChanges.UnchangedPaths.Keys, SpecFile));
});
AssertLogged(LogEventId.EndSerializingPipGraph);
hostController.Dispose();
}
}
protected BuildXLEngineResult RunEngineWithConfig(ICommandLineConfiguration config, TestCache testCache = null)
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// Set the specified phase
((CommandLineConfiguration)config).Engine.Phase = Phase;
var engineResult = CreateAndRunEngine(
config,
appDeployment,
testRootDirectory: null,
rememberAllChangedTrackedInputs: true,
engine: out var engine,
testCache: testCache);
return(engineResult);
}
}
public void FileProbShouldNotInvalidateTheCacheIfAnotherFileWasAdded()
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// The same testRootDirectory is used for all invocations, so the spec cache can be reused.
var testRoot = tempFiles.GetUniqueDirectory(PathTable).ToString(PathTable);
string spec = @"export const r = File.exists(f`../foo/a.txt`);";
var buildDefinition = CreateDefinition(spec);
RunAndAssertGraphCacheMiss(WriteSpecs(testRoot, buildDefinition), appDeployment);
// Adding a random file (that was not probed) should not affect the cache.
buildDefinition["../foo/b.txt"] = "";
RunAndAssertGraphCacheHit(WriteSpecs(testRoot, buildDefinition), appDeployment);
}
}
/// <summary>
/// Runs the engine for a given config, assuming the Yarn repo is already initialized
/// </summary>
protected BuildXLEngineResult RunEngine(ICommandLineConfiguration config, TestCache testCache = null, IDetoursEventListener detoursListener = null)
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
((CommandLineConfiguration)config).Engine.Phase = Phase;
((CommandLineConfiguration)config).Sandbox.FileSystemMode = FileSystemMode.RealAndMinimalPipGraph;
var engineResult = CreateAndRunEngine(
config,
appDeployment,
testRootDirectory: null,
rememberAllChangedTrackedInputs: true,
engine: out var engine,
testCache: testCache,
detoursListener: detoursListener);
return(engineResult);
}
}
public async Task OutputInMemory()
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var storage = (ISandboxedProcessFileStorage)tempFiles;
var fileName = storage.GetFileName(SandboxedProcessFile.StandardOutput);
var content = new string('S', 100);
var outputBuilder =
new SandboxedProcessOutputBuilder(
Encoding.UTF8,
content.Length + Environment.NewLine.Length,
tempFiles,
SandboxedProcessFile.StandardOutput,
null);
outputBuilder.AppendLine(content);
var output = outputBuilder.Freeze();
XAssert.IsFalse(output.IsSaved);
XAssert.IsFalse(File.Exists(fileName));
XAssert.AreEqual(content + Environment.NewLine, await output.ReadValueAsync());
}
}
public void FileProbShouldInvalidateTheCache()
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// The same testRootDirectory is used for all invocations, so the spec cache can be reused.
var testRoot = tempFiles.GetUniqueDirectory(PathTable).ToString(PathTable);
// Spec probs the file existence.
string spec = @"export const r = File.exists(f`../foo/a.txt`);";
var buildDefinition = CreateDefinition(spec);
RunAndAssertGraphCacheMiss(WriteSpecs(testRoot, buildDefinition), appDeployment);
// Adding a probbed file should lead to a cache miss
buildDefinition["../foo/a.txt"] = "";
RunAndAssertGraphCacheMiss(WriteSpecs(testRoot, buildDefinition), appDeployment);
}
}
public void PublicFacadeAndAstAreNotReusedForSpecsWithPublicSurfaceChanges()
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// The same testRootDirectory is used for both invocations, so the spec cache can be reused.
var testRoot = tempFiles.GetUniqueDirectory(PathTable).ToString(PathTable);
var config = Build()
.EmptyConfiguration()
.TestRootDirectory(testRoot)
.AddSpec("spec1.dsc", "export const x = 42;")
.AddSpec("spec2.dsc", "export const y = x;")
.AddSpec("spec3.dsc", "export const w = y;")
.AddSpec("spec4.dsc", "export const z = w;")
.AddSpec("spec5.dsc", "export const notUsedYet = 42;")
.PersistSpecsAndGetConfiguration(enableSpecCache: true);
RunAndRetrieveSpecs(config, appDeployment);
// Changing the declartion fingerprint for the spec 3.
config = Build()
.EmptyConfiguration()
.TestRootDirectory(testRoot)
.AddSpec("spec3.dsc", "export const w = y; export const another = 42;; //unsafe change")
.PersistSpecsAndGetConfiguration(cleanExistingDirectory: false, enableSpecCache: true);
// Now all the specs should be fully parsed.
using (var controller = RunEngineAndGetFrontEndHostController(config, appDeployment, testRoot, rememberAllChangedTrackedInputs: true))
{
var workspace = controller.Workspace;
var sourceFiles = workspace.GetAllSourceFiles();
foreach (var sourceFile in sourceFiles)
{
Assert.False(sourceFile.IsPublicFacade, $"{sourceFile.Path.AbsolutePath} should not be a public facade.");
}
}
}
}
public void ChangingMountsGraphCacheMiss()
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// The same testRootDirectory is used for all invocations, so the spec cache can be reused.
var testRoot = tempFiles.GetUniqueDirectory(PathTable).ToString(PathTable);
string spec = @"export const r = File.exists(f`${Context.getMount(""ObjectRoot"").path}/foo/a.txt`);";
var buildDefinition = CreateDefinition(spec);
var config = WriteSpecs(testRoot, buildDefinition);
RunAndAssertGraphCacheMiss(config, appDeployment);
var mutableConfig = (CommandLineConfiguration)config;
RunAndAssertGraphCacheHit(mutableConfig, appDeployment);
// Changing the path for a mount should cause a graph cache miss
mutableConfig.Layout.ObjectDirectory = tempFiles.GetUniqueDirectory(PathTable);
RunAndAssertGraphCacheMiss(mutableConfig, appDeployment);
}
}
protected BuildXLEngineResult RunEngineWithConfig(ICommandLineConfiguration config, TestCache testCache = null)
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// Set the specified phase
((CommandLineConfiguration)config).Engine.Phase = Phase;
((CommandLineConfiguration)config).Sandbox.FileSystemMode = FileSystemMode.RealAndMinimalPipGraph;
// Source resolver not properly registered for Ninja tests
((CommandLineConfiguration)config).DisableInBoxSdkSourceResolver = true;
var engineResult = CreateAndRunEngine(
config,
appDeployment,
testRootDirectory: null,
rememberAllChangedTrackedInputs: true,
engine: out _,
testCache: testCache);
return(engineResult);
}
}
/// <summary>
/// Runs the analyzer on the testSouce and expects it to succeed with the given formatted text.
/// </summary>
public void DocumentationGeneratorTestSuccess(
string testSource,
string expectedFileName = null,
string expectedFileContents = null,
string[] extraSources = null,
Dictionary <string, string> modules = null)
{
using (TempFileStorage temp = new TempFileStorage(true, TemporaryDirectory))
{
TestHelper(
testSource,
extraSources,
modules,
new[] { "/outputFolder:" + temp.RootDirectory, "/rootLink:BuildXL/Reference-Guide/Sdk-Documentation" },
false,
(success, logger, sourceFile) =>
{
var errors = string.Join(Environment.NewLine, logger.CapturedDiagnostics.Select(d => d.Message));
Assert.True(success, "Expect to have successful run. Encountered:\r\n" + errors);
Assert.False(logger.HasErrors, "Expect to have no errors. Encountered:\r\n" + errors);
string contents = File.ReadAllText(Path.Combine(temp.RootDirectory, expectedFileName));
Console.Write(expectedFileName + ":");
Console.Write(contents);
if (OperatingSystemHelper.IsUnixOS)
{
// This file is saved as Windows CRLF, hence convert on Unix systems for the tests to pass
contents = contents.Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", "\r\n");
}
Assert.Equal(expectedFileContents, contents);
},
true);
}
}
public void RemovingTheFileShouldInvalidateTheCache()
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// The same testRootDirectory is used for all invocations, so the spec cache can be reused.
var testRoot = tempFiles.GetUniqueDirectory(PathTable).ToString(PathTable);
// Spec probs the file existence.
string spec = @"export const r = File.exists(f`foo/a.txt`);";
var buildDefinition = CreateDefinition(spec);
buildDefinition["foo/a.txt"] = "1";
RunAndAssertGraphCacheMiss(WriteSpecs(testRoot, buildDefinition), appDeployment);
// Creating another definition without "foo/a.txt".
// but since we are reusing the same testroot we have to delete thefile that is going missing.
File.Delete(Path.Combine(this.RelativeSourceRoot, "Foo", "a.txt"));
// This should lead to a cache miss.
buildDefinition = CreateDefinition(spec);
RunAndAssertGraphCacheMiss(WriteSpecs(testRoot, buildDefinition), appDeployment);
}
}
public void TestInputChangesTriggerGraphCacheMiss(BuildDefinition buildDefinition)
{
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TestOutputDirectory))
{
var appDeployment = CreateAppDeployment(tempFiles);
// The same testRootDirectory is used for all invocations, so the spec cache can be reused.
var testRoot = tempFiles.GetUniqueDirectory(PathTable).ToString(PathTable);
// run once to begin with and assert cache miss
RunAndAssertGraphCacheMiss(WriteSpecs(testRoot, buildDefinition), appDeployment);
// for each spec
foreach (var specTuple in buildDefinition.Specs)
{
// run again without changing anything and assert graph cache hit
RunAndAssertGraphCacheHit(WriteSpecs(testRoot, buildDefinition), appDeployment);
// change selected spec and assert graph cache miss
buildDefinition.AppendToSpecContent(specPath: specTuple.SpecPath, contentToAppend: " ");
RunAndAssertGraphCacheMiss(WriteSpecs(testRoot, buildDefinition), appDeployment);
}
}
}
/// <summary>
/// Creates a default configuration
/// </summary>
public static CommandLineConfiguration CreateDefault(PathTable pathTable, string configFilePath, TempFileStorage tempStorage)
{
var paths = new Paths(pathTable);
var configFile = paths.CreateAbsolutePath(configFilePath);
var result = ConfigurationHelpers.GetDefaultForTesting(pathTable, configFile);
result.Layout.ObjectDirectory = paths.CreateAbsolutePath(tempStorage.GetUniqueDirectory());
result.Layout.CacheDirectory = paths.CreateAbsolutePath(tempStorage.GetUniqueDirectory());
return(result);
}
