private void SetProcessEnvironmentVariables(IReadOnlyDictionary <string, string> environment, ProcessBuilder processBuilder)
{
foreach (KeyValuePair <string, string> kvp in environment)
{
if (kvp.Value != null)
{
var envPipData = new PipDataBuilder(m_context.StringTable);
// Casing for paths is not stable as reported by BuildPrediction. So here we try to guess if the value
// represents a path, and normalize it
string value = kvp.Value;
if (!string.IsNullOrEmpty(value) && AbsolutePath.TryCreate(PathTable, value, out var absolutePath))
{
envPipData.Add(absolutePath);
}
else
{
envPipData.Add(value);
}
processBuilder.SetEnvironmentVariable(
StringId.Create(m_context.StringTable, kvp.Key),
envPipData.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
}
}
if (m_userDefinedPassthroughVariables != null)
{
foreach (string passThroughVariable in m_userDefinedPassthroughVariables)
{
processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, passThroughVariable));
}
}
}
public PipDataTests(ITestOutputHelper output)
: base(output)
{
m_pathTable = new PathTable();
m_expectedStringId0 = StringId.Create(m_pathTable.StringTable, ExpectedString0);
m_pipDataBuilder = new PipDataBuilder(m_pathTable.StringTable);
m_expectedStringId0 = StringId.Create(m_pathTable.StringTable, ExpectedString0);
m_uniqueEntry0 = AbsolutePath.Create(m_pathTable, A("c", "unique to fragment 0"));
// BEGIN ADDING ARGUMENTS
m_cursorStart = m_pipDataBuilder.CreateCursor();
AddStandardBlock(m_pipDataBuilder);
m_cursor0 = m_pipDataBuilder.CreateCursor();
using (m_pipDataBuilder.StartFragment(Escaping0, m_separator0))
{
m_pipDataBuilder.Add(m_uniqueEntry0);
AddStandardBlock(m_pipDataBuilder);
}
m_cursor1 = m_pipDataBuilder.CreateCursor();
using (m_pipDataBuilder.StartFragment(Escaping1, Separator1))
{
AddStandardBlock(m_pipDataBuilder);
m_pipDataBuilder.Add(UniqueEntry1);
}
m_cursorEnd = m_pipDataBuilder.CreateCursor();
// END ADDING ARGUMENTS
}
public void ExecutionProcessReadingStdIn()
{
FileArtifact stdOut = CreateOutputFileArtifact();
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadStdIn() });
PipDataBuilder dataBuilder = new PipDataBuilder(Context.PathTable.StringTable);
dataBuilder.Add("Data0");
dataBuilder.Add("Data1");
dataBuilder.Add("Data2");
builder.StandardInput = global::BuildXL.Pips.StandardInput.CreateFromData(dataBuilder.ToPipData(Environment.NewLine, PipDataFragmentEscaping.NoEscaping));
builder.SetStandardOutputFile(stdOut.Path);
builder.Options |= Process.Options.RequiresAdmin;
ProcessWithOutputs process = SchedulePipBuilder(builder);
RunScheduler().AssertSuccess();
string[] output = File.ReadAllLines(ArtifactToString(stdOut));
string actualContent = string.Join(Environment.NewLine, output);
XAssert.AreEqual(3, output.Length, "Actual content: {0}{1}", Environment.NewLine, string.Join(Environment.NewLine, output));
for (int i = 0; i < 3; ++i)
{
XAssert.AreEqual("Data" + i, output[i], "Actual content: {0}", output[i]);
}
}
private static void AddLogArgument(PipDataBuilder pipDataBuilder, int loggerNumber, AbsolutePath logFile, string verbosity)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString(I($"/flp{loggerNumber}:logfile=")));
pipDataBuilder.Add((PipDataAtom.FromAbsolutePath(logFile)));
pipDataBuilder.Add(PipDataAtom.FromString(I($";{verbosity}")));
}
}
private const int NumStandardBlockFragments = 8; // this number must match the number of fragments added in AddStandardBlock
private void AddStandardBlock(PipDataBuilder pipDataBuilder)
{
pipDataBuilder.Add(Path1);
pipDataBuilder.Add(Path2);
pipDataBuilder.Add(CaseVaryingString);
pipDataBuilder.Add(m_expectedStringId0);
pipDataBuilder.AddVsoHash(SourceFile);
pipDataBuilder.AddVsoHash(OutputFile);
pipDataBuilder.AddVsoHash(RewrittenFile);
pipDataBuilder.AddVsoHash(RewrittenFile2);
// if you add more fragments here, update
// (1) NumStandardBlockFragments constant above
// (2) VerifyStandardBlock method below
}
public static void Add(this PipDataBuilder @this, string prefix, RelativePath value)
{
var escaping = PipDataFragmentEscaping.CRuntimeArgumentRules;
var separator = System.IO.Path.DirectorySeparatorChar.ToString();
using (@this.StartFragment(escaping, separator))
{
if (!string.IsNullOrEmpty(prefix))
{
@this.Add(prefix);
}
foreach (var atom in value.GetAtoms())
{
@this.Add(atom);
}
}
}
private static void AddMsBuildProperty(PipDataBuilder pipDataBuilder, string key, string value)
{
// Make sure properties are always quoted, since MSBuild doesn't handle semicolon separated
// properties without quotes
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString("/p:"));
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString(key));
}
pipDataBuilder.Add(PipDataAtom.FromString("=\""));
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString(value));
}
pipDataBuilder.Add(PipDataAtom.FromString("\""));
}
}
/// <summary>
/// Creates arguments.
/// </summary>
protected void CreateArguments(
PipDataBuilder pipDataBuilder,
IEnumerable <Operation> processOperations,
StringTable stringTable)
{
foreach (var op in processOperations)
{
pipDataBuilder.Add(op.ToCommandLine(Context.PathTable));
}
}
/// <summary>
/// Configure the environment for a process
/// </summary>
public static void SetProcessEnvironmentVariables(
IReadOnlyDictionary <string, string> userDefinedEnvironment,
[CanBeNull] IEnumerable <string> userDefinedPassthroughVariables,
ProcessBuilder processBuilder,
PathTable pathTable)
{
Contract.RequiresNotNull(userDefinedEnvironment);
Contract.RequiresNotNull(processBuilder);
foreach (KeyValuePair <string, string> kvp in userDefinedEnvironment)
{
if (kvp.Value != null)
{
var envPipData = new PipDataBuilder(pathTable.StringTable);
// Casing for paths is not stable as reported by BuildPrediction. So here we try to guess if the value
// represents a path, and normalize it
string value = kvp.Value;
if (!string.IsNullOrEmpty(value) && AbsolutePath.TryCreate(pathTable, value, out var absolutePath))
{
envPipData.Add(absolutePath);
}
else
{
envPipData.Add(value);
}
processBuilder.SetEnvironmentVariable(
StringId.Create(pathTable.StringTable, kvp.Key),
envPipData.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
}
}
if (userDefinedPassthroughVariables != null)
{
foreach (string passThroughVariable in userDefinedPassthroughVariables)
{
processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(pathTable.StringTable, passThroughVariable));
}
}
}
private void AddOption <TValue>(string optionName, TValue value, Action <PipDataBuilder, TValue> writeValue)
{
Contract.Requires(!string.IsNullOrEmpty(optionName));
Contract.Requires(writeValue != null);
Contract.Assert(!m_finished);
using (m_builder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, string.Empty))
{
m_builder.Add(optionName);
writeValue(m_builder, value);
}
}
private void SetEnvironmentVariables(ProcessBuilder processBuilder, NinjaNode node)
{
foreach (KeyValuePair <string, string> kvp in m_environmentVariables.Value)
{
if (kvp.Value != null)
{
var envPipData = new PipDataBuilder(m_context.StringTable);
if (SpecialEnvironmentVariables.PassThroughPrefixes.All(prefix => !kvp.Key.StartsWith(prefix)))
{
envPipData.Add(kvp.Value);
processBuilder.SetEnvironmentVariable(StringId.Create(m_context.StringTable, kvp.Key), envPipData.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
}
}
}
foreach (var kvp in m_passThroughEnvironmentVariables.Value)
{
processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, kvp.Key));
}
foreach (var envVar in SpecialEnvironmentVariables.CloudBuildEnvironment)
{
processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, envVar));
}
// GlobalUnsafePassthroughEnvironmentVariables
processBuilder.SetGlobalPassthroughEnvironmentVariable(m_frontEndHost.Configuration.FrontEnd.GlobalUnsafePassthroughEnvironmentVariables, m_context.StringTable);
// We will specify a different MSPDBSRV endpoint for every pip.
// This means every pip that needs to communicate to MSPDBSRV will
// spawn a different child process.
// This is because if two pips use the same endpoint at the same time
// then second one will fail after the first one finishes, because the
// first one was the one that spawned MSPDBSRV.EXE as a child
// (which gets killed).
//
// IMPORTANT: This will cause the build to fail if two pips target the same PDB file.
// Both LINK.EXE and CL.EXE can use MSPDBSRV.EXE:
// - If all linkers specify a different /pdb argument (or don't specify this argument and
// are linking programs with different names
// [https://docs.microsoft.com/en-us/cpp/build/reference/debug-generate-debug-info])
// then this won't happen
//
// - We're forcing the compiler to not output to PDBs (/Z7)
//
// so this should work in the normal cases.
var mspdbsrvPipDataBuilder = new PipDataBuilder(m_context.StringTable);
mspdbsrvPipDataBuilder.Add(PipConstructionUtilities.ComputeSha256(node.Command)); // Unique value for each pip
processBuilder.SetEnvironmentVariable(
StringId.Create(m_context.StringTable, SpecialEnvironmentVariables.MsPdvSrvEndpoint),
mspdbsrvPipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
}
private void AddOption <TValue>(string prefix, TValue value, bool valueIsEmpty, Action <PipDataBuilder, TValue> writeValue)
{
// prefix and value are null -> skip
if (string.IsNullOrEmpty(prefix) && valueIsEmpty)
{
return;
}
if (string.IsNullOrEmpty(prefix))
{
// This is unnamed argument
writeValue(ArgumentsBuilder, value);
return;
}
if (valueIsEmpty)
{
// This is a flag (or switch) kind of arguments
ArgumentsBuilder.Add(prefix);
return;
}
// both prefix and value are non-empty
// - handle the special case when prefix ends with space
// -> (1) add trimmed prefix, (2) add raw space, (3) add value
if (prefix.EndsWith(" ", StringComparison.OrdinalIgnoreCase))
{
AddRawText(prefix.TrimEnd(' '));
writeValue(ArgumentsBuilder, value);
return;
}
using (ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, EmptyStringId))
{
ArgumentsBuilder.Add(prefix);
writeValue(ArgumentsBuilder, value);
}
}
private void AddJavaScriptArgumentToBuilder(PipDataBuilder argumentsBuilder, JavaScriptArgument value)
{
switch (value.GetValue())
{
case string s:
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, m_context.StringTable.Empty))
{
argumentsBuilder.Add(s);
}
break;
case AbsolutePath absolutePath:
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, m_context.StringTable.Empty))
{
argumentsBuilder.Add(absolutePath);
}
break;
case RelativePath relativePath:
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, m_context.StringTable.Empty))
{
argumentsBuilder.Add(relativePath);
}
break;
case PathAtom pathAtom:
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, m_context.StringTable.Empty))
{
argumentsBuilder.Add(pathAtom);
}
break;
default:
Contract.Assert(false, $"Unexpected argument '{value.GetType()}'");
break;
}
}
public FileArtifact WriteFile(AbsolutePath destination, PipDataAtom content, WriteFileEncoding?encoding = null)
{
PipDataBuilder pipDataBuilder = new PipDataBuilder(Context.StringTable);
pipDataBuilder.Add(content);
if (!PipConstructionHelper.TryWriteFile(
destination,
pipDataBuilder.ToPipData(Context.StringTable.Empty, PipDataFragmentEscaping.NoEscaping),
encoding ?? WriteFileEncoding.Utf8,
null,
null,
out var result))
{
throw new BuildXLTestException("Failed to add writefile pip");
}
return(result);
}
/// <summary>
/// Gets the command line arguments for the process.
/// </summary>
public PipData GetArgumentsDataFromProcess(Process process)
{
PipData arguments = process.Arguments;
if (process.ResponseFile.IsValid)
{
var responseFileData = process.ResponseFileData;
PipDataBuilder pipDataBuilder = new PipDataBuilder(StringTable);
// Add all the arguments from the command line excluding the response file (the last fragment)
foreach (var fragment in process.Arguments.Take(process.Arguments.FragmentCount - 1).Concat(responseFileData))
{
Contract.Assume(fragment.FragmentType != PipFragmentType.Invalid);
pipDataBuilder.Add(fragment);
}
arguments = pipDataBuilder.ToPipData(arguments.FragmentSeparator, arguments.FragmentEscaping);
}
return(arguments);
}
private bool TryAddMsBuildArguments(ProjectWithPredictions <AbsolutePath> project, Qualifier qualifier, PipDataBuilder pipDataBuilder, AbsolutePath logDirectory, out string failureDetail)
{
// Common arguments to all MsBuildExe invocations
pipDataBuilder.AddRange(s_commonArgumentsToMsBuildExe.Select(argument => PipDataAtom.FromString(argument)));
// Log verbosity
if (!TryGetLogVerbosity(m_resolverSettings.LogVerbosity, out string logVerbosity))
{
failureDetail = $"Cannot set the MSBuild log verbosity. '{m_resolverSettings.LogVerbosity}' is not a valid option.";
return(false);
}
AddLogArgument(pipDataBuilder, 1, logDirectory.Combine(PathTable, "msbuild.log"), $"Verbosity={logVerbosity}");
AddLogArgument(pipDataBuilder, 2, logDirectory.Combine(PathTable, "msbuild.wrn"), "Verbosity=Quiet;warningsonly");
AddLogArgument(pipDataBuilder, 3, logDirectory.Combine(PathTable, "msbuild.err"), "Verbosity=Quiet;errorsonly");
AddLogArgument(pipDataBuilder, 4, logDirectory.Combine(PathTable, "msbuild.prf"), "PerformanceSummary");
// Global properties on the project are turned into build parameters
foreach (var kvp in project.GlobalProperties)
{
AddMsBuildProperty(pipDataBuilder, kvp.Key, kvp.Value);
}
// The specified qualifier, unless overridden by a global property, is turned into a property as well
// This means that:
// 1) if the project is not being referenced with a specific property that matters to the qualifier, the requested qualifier is used
// 2) if a particular property (e.g. platform) is set when referencing the project, that is honored
foreach (StringId key in qualifier.Keys)
{
string keyAsString = key.ToString(m_context.StringTable);
if (!project.GlobalProperties.ContainsKey(keyAsString))
{
var success = qualifier.TryGetValue(m_context.StringTable, keyAsString, out string value);
Contract.Assert(success);
AddMsBuildProperty(pipDataBuilder, keyAsString, value);
}
}
// Configure binary logger if specified
if (m_resolverSettings?.EnableBinLogTracing == true)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString("/binaryLogger:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(logDirectory.Combine(PathTable, "msbuild.binlog")));
}
}
// Targets to execute.
// If the prediction is available, there should be at least one target (otherwise it makes no sense to schedule the project, and we should have caught this earlier)
// If the prediction is not available, we fallback to call default targets (which means not passing any specific /t:). A more strict policy would be to bail out
// here saying that the project is not complying to the target protocol specification. We leave it relaxed for now, but we log it.
// https://github.com/Microsoft/msbuild/blob/master/documentation/specs/static-graph.md
if (project.PredictedTargetsToExecute.IsPredictionAvailable)
{
var targets = project.PredictedTargetsToExecute.Targets;
Contract.Assert(targets.Count > 0);
foreach (string target in targets)
{
pipDataBuilder.Add(PipDataAtom.FromString($"/t:{target}"));
}
}
else
{
// The prediction for the targets to execute is not available. Just log this as a warning for now, defaults targets will be used.
Tracing.Logger.Log.ProjectIsNotSpecifyingTheProjectReferenceProtocol(
m_context.LoggingContext,
Location.FromFile(project.FullPath.ToString(PathTable)),
project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable));
}
failureDetail = string.Empty;
return(true);
}
private static Process CreateConsoleProcessInContainer(
BuildXLContext context,
TempFileStorage tempFiles,
PathTable pt,
string arguments,
ReadOnlyArray <FileArtifactWithAttributes> outputFiles,
ReadOnlyArray <DirectoryArtifact> directoryOutputs,
ContainerIsolationLevel containerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs)
{
var executableFileArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(context.PathTable, CmdHelper.CmdX64));
var argumentBuilder = new PipDataBuilder(context.PathTable.StringTable);
argumentBuilder.Add("/d");
argumentBuilder.Add("/c");
using (argumentBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
foreach (var arg in arguments.Split(new[] { ' ' }, System.StringSplitOptions.RemoveEmptyEntries))
{
argumentBuilder.Add(arg);
}
}
string workingDirectory = tempFiles.GetUniqueDirectory();
var workingDirectoryAbsolutePath = AbsolutePath.Create(context.PathTable, workingDirectory);
string uniqueOutputDirectory = tempFiles.GetUniqueDirectory();
var uniqueOutputDirectoryPath = AbsolutePath.Create(context.PathTable, uniqueOutputDirectory);
string uniqueRedirectedOutputDirectory = tempFiles.GetUniqueDirectory("redirected");
var uniqueRedirectedOutputDirectoryPath = AbsolutePath.Create(context.PathTable, uniqueRedirectedOutputDirectory);
var pip = new Process(
executableFileArtifact,
workingDirectoryAbsolutePath,
argumentBuilder.ToPipData(" ", PipDataFragmentEscaping.NoEscaping),
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .FromWithoutCopy(),
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pt),
null,
null,
dependencies: ReadOnlyArray <FileArtifact> .FromWithoutCopy(new[] { executableFileArtifact }),
outputs: outputFiles,
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty,
directoryOutputs: directoryOutputs,
orderDependencies: ReadOnlyArray <PipId> .Empty,
untrackedPaths: ReadOnlyArray <AbsolutePath> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .Empty,
tags: ReadOnlyArray <StringId> .Empty,
successExitCodes: ReadOnlyArray <int> .Empty,
semaphores: ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty,
options: Process.Options.NeedsToRunInContainer,
uniqueOutputDirectory: uniqueOutputDirectoryPath,
uniqueRedirectedDirectoryRoot: uniqueRedirectedOutputDirectoryPath,
containerIsolationLevel: containerIsolationLevel);
return(pip);
}
/// <summary>
/// Creates arguments.
/// </summary>
internal static PipData CreateCmdArguments(
StringTable stringTable,
IEnumerable <FileArtifact> dependencies,
IEnumerable <FileArtifact> outputs,
bool includeWarning = false)
{
Contract.Requires(dependencies != null, "Argument dependencies cannot be null");
Contract.Requires(outputs != null, "Argument outputs cannot be null");
var pipDataBuilder = new PipDataBuilder(stringTable);
int i = 0;
pipDataBuilder.Add("/d");
pipDataBuilder.Add("/c");
foreach (FileArtifact output in outputs)
{
if (i > 0)
{
pipDataBuilder.Add("&");
}
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
var allDependencies = dependencies.ToList();
pipDataBuilder.Add("type");
foreach (FileArtifact dependency in allDependencies)
{
pipDataBuilder.Add(dependency);
}
pipDataBuilder.Add(">");
pipDataBuilder.Add(output);
}
pipDataBuilder.Add("&");
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
pipDataBuilder.Add("echo");
pipDataBuilder.Add("buildxl");
pipDataBuilder.Add(">>");
pipDataBuilder.Add(output);
}
i++;
}
if (includeWarning)
{
if (outputs.Any())
{
pipDataBuilder.Add("&&");
}
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
pipDataBuilder.Add("echo");
pipDataBuilder.Add(WarningRegexDescription);
}
}
return(pipDataBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules));
}
public static void Add(this PipDataBuilder @this, RelativePath value) => @this.Add(null, value);
private bool TryAddMsBuildArguments(ProjectWithPredictions project, PipDataBuilder pipDataBuilder, AbsolutePath logDirectory, AbsolutePath outputResultCacheFile, out string failureDetail)
{
// Common arguments to all MsBuildExe invocations
pipDataBuilder.AddRange(s_commonArgumentsToMsBuildExe.Select(argument => PipDataAtom.FromString(argument)));
// Log verbosity
if (!TryGetLogVerbosity(m_resolverSettings.LogVerbosity, out string logVerbosity))
{
failureDetail = $"Cannot set the MSBuild log verbosity. '{m_resolverSettings.LogVerbosity}' is not a valid option.";
return(false);
}
AddLogArgument(pipDataBuilder, 1, logDirectory.Combine(PathTable, "msbuild.log"), $"Verbosity={logVerbosity}");
AddLogArgument(pipDataBuilder, 2, logDirectory.Combine(PathTable, "msbuild.wrn"), "Verbosity=Quiet;warningsonly");
AddLogArgument(pipDataBuilder, 3, logDirectory.Combine(PathTable, "msbuild.err"), "Verbosity=Quiet;errorsonly");
AddLogArgument(pipDataBuilder, 4, logDirectory.Combine(PathTable, "msbuild.prf"), "PerformanceSummary");
// Global properties on the project are turned into build parameters
foreach (var kvp in project.GlobalProperties)
{
AddMsBuildProperty(pipDataBuilder, kvp.Key, kvp.Value);
}
// Configure binary logger if specified
if (m_resolverSettings.EnableBinLogTracing == true)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString("/binaryLogger:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(logDirectory.Combine(PathTable, "msbuild.binlog")));
}
}
// Targets to execute.
var targets = project.PredictedTargetsToExecute.Targets;
Contract.Assert(targets.Count > 0);
foreach (string target in targets)
{
pipDataBuilder.Add(PipDataAtom.FromString($"/t:{target}"));
}
// Pass the output result cache file if present
if (outputResultCacheFile != AbsolutePath.Invalid)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
// Flag /orc is the short form of /outputResultsCache, and part of MSBuild 'build in isolation' mode.
// By specifying this flag, MSBuild will write the build result at the end of this invocation into the cache file
pipDataBuilder.Add(PipDataAtom.FromString("/orc:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(outputResultCacheFile));
}
}
else
{
// In legacy (non-isolated) mode, we still have to rely on SDKs honoring this flag
pipDataBuilder.Add(PipDataAtom.FromString("/p:buildprojectreferences=false"));
}
failureDetail = string.Empty;
return(true);
}
private async Task ProcessWindowsCallHelper(
string functionName,
SandboxConfiguration config = null,
IEnumerable <string> extraDependencies = null,
IEnumerable <string> extraOutputs = null,
int callCount = 1,
string commandPrefix = "",
bool readsAndWritesDirectories = false,
bool untrackedOutputs = false,
string[] expectedWarningStrings = null,
string[] expectedErrorStrings = null)
{
if (config == null)
{
config = new SandboxConfiguration {
FileAccessIgnoreCodeCoverage = true, FailUnexpectedFileAccesses = true
};
}
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var fileContentTable = FileContentTable.CreateNew(LoggingContext);
// have to force the config for truncation
config.OutputReportingMode = OutputReportingMode.FullOutputOnWarningOrError;
bool expectSuccess = expectedErrorStrings == null && expectedWarningStrings == null;
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
{
string currentCodeFolder = Path.GetDirectoryName(AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly()));
Contract.Assume(currentCodeFolder != null);
string executable = Path.Combine(currentCodeFolder, DetourTestFolder, "DetoursTests.exe");
string workingDirectory = tempFiles.GetUniqueDirectory();
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
XAssert.IsTrue(File.Exists(executable), "Could not find the test file: " + executable);
FileArtifact executableFileArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
var extraUntrackedScopes = new List <AbsolutePath>();
var dependencies = new List <FileArtifact> {
executableFileArtifact
};
if (extraDependencies != null)
{
foreach (string file in extraDependencies)
{
string filePath = Path.Combine(workingDirectory, file);
AbsolutePath path = AbsolutePath.Create(pathTable, filePath);
if (readsAndWritesDirectories)
{
Directory.CreateDirectory(filePath);
// We don't support directories as inputs in BuildXL yet.
extraUntrackedScopes.Add(path);
}
else
{
File.WriteAllText(filePath, "Definitely a file");
FileArtifact fileArtifact = FileArtifact.CreateSourceFile(path);
dependencies.Add(fileArtifact);
}
}
}
var outputs = new List <FileArtifactWithAttributes>();
if (extraOutputs != null)
{
foreach (string file in extraOutputs)
{
string filePath = Path.Combine(workingDirectory, file);
AbsolutePath path = AbsolutePath.Create(pathTable, filePath);
if (readsAndWritesDirectories)
{
// We don't support directory outputs in BuildXL at the moment, so e.g. deleting a directory needs to be untracked.
extraUntrackedScopes.Add(path);
}
else if (untrackedOutputs)
{
extraUntrackedScopes.Add(path);
}
else
{
FileArtifact fileArtifact = FileArtifact.CreateSourceFile(path).CreateNextWrittenVersion();
outputs.Add(fileArtifact.WithAttributes());
}
}
}
var tempDirectory = tempFiles.GetUniqueDirectory();
var environmentVariables = new List <EnvironmentVariable>();
var environmentValue = new PipDataBuilder(pathTable.StringTable);
var tempPath = AbsolutePath.Create(pathTable, tempDirectory);
environmentValue.Add(tempPath);
environmentVariables.Add(new EnvironmentVariable(StringId.Create(pathTable.StringTable, "TMP"), environmentValue.ToPipData(" ", PipDataFragmentEscaping.NoEscaping)));
environmentVariables.Add(new EnvironmentVariable(StringId.Create(pathTable.StringTable, "TEMP"), environmentValue.ToPipData(" ", PipDataFragmentEscaping.NoEscaping)));
var untrackedPaths = CmdHelper.GetCmdDependencies(pathTable);
var untrackedScopes = extraUntrackedScopes.Concat(CmdHelper.GetCmdDependencyScopes(pathTable).Concat(new[] { tempPath })).Distinct();
var pip = new Process(
executableFileArtifact,
workingDirectoryAbsolutePath,
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.NoEscaping, commandPrefix + functionName + "Logging"),
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .From(environmentVariables),
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pathTable),
null,
null,
ReadOnlyArray <FileArtifact> .From(dependencies),
ReadOnlyArray <FileArtifactWithAttributes> .From(outputs),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(untrackedPaths),
ReadOnlyArray <AbsolutePath> .From(untrackedScopes),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
if (expectSuccess)
{
await AssertProcessSucceedsAsync(
context,
config,
pip);
}
else
{
await AssertProcessCompletesWithStatusAsync(
SandboxedProcessPipExecutionStatus.ExecutionFailed,
context,
config,
pip,
null);
}
}
int expectedErrorCount = 0;
int expectedWarningCount = 0;
IEnumerable <string> requiredLogMessageSubstrings = new string[] { };
if (expectedErrorStrings != null)
{
expectedErrorCount = expectedErrorStrings.Count();
requiredLogMessageSubstrings = requiredLogMessageSubstrings.Concat(expectedErrorStrings);
}
if (expectedWarningStrings != null)
{
expectedWarningCount = expectedWarningStrings.Count();
requiredLogMessageSubstrings = requiredLogMessageSubstrings.Concat(expectedWarningStrings);
}
SetExpectedFailures(expectedErrorCount, expectedWarningCount, requiredLogMessageSubstrings.ToArray());
}
private bool TryAddMsBuildArguments(ProjectWithPredictions project, PipDataBuilder pipDataBuilder, AbsolutePath logDirectory, AbsolutePath outputResultCacheFile, out string failureDetail)
{
// Common arguments to all MsBuildExe invocations
pipDataBuilder.AddRange(s_commonArgumentsToMsBuildExe.Select(argument => PipDataAtom.FromString(argument)));
// Log verbosity
if (!TryGetLogVerbosity(m_resolverSettings.LogVerbosity, out string logVerbosity))
{
failureDetail = $"Cannot set the MSBuild log verbosity. '{m_resolverSettings.LogVerbosity}' is not a valid option.";
return(false);
}
AddLogArgument(pipDataBuilder, 1, logDirectory.Combine(PathTable, "msbuild.log"), $"Verbosity={logVerbosity}");
AddLogArgument(pipDataBuilder, 2, logDirectory.Combine(PathTable, "msbuild.wrn"), "Verbosity=Quiet;warningsonly");
AddLogArgument(pipDataBuilder, 3, logDirectory.Combine(PathTable, "msbuild.err"), "Verbosity=Quiet;errorsonly");
AddLogArgument(pipDataBuilder, 4, logDirectory.Combine(PathTable, "msbuild.prf"), "PerformanceSummary");
// Global properties on the project are turned into build parameters
foreach (var kvp in project.GlobalProperties)
{
AddMsBuildProperty(pipDataBuilder, kvp.Key, kvp.Value);
}
// Configure binary logger if specified
if (m_resolverSettings.EnableBinLogTracing == true)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString("/binaryLogger:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(logDirectory.Combine(PathTable, "msbuild.binlog")));
}
}
// Targets to execute.
// If the prediction is available, there should be at least one target (otherwise it makes no sense to schedule the project, and we should have caught this earlier)
// If the prediction is not available, we fallback to call default targets (which means not passing any specific /t:). A more strict policy would be to bail out
// here saying that the project is not complying to the target protocol specification. We leave it relaxed for now, but we log it.
// https://github.com/Microsoft/msbuild/blob/master/documentation/specs/static-graph.md
if (project.PredictedTargetsToExecute.IsPredictionAvailable)
{
var targets = project.PredictedTargetsToExecute.Targets;
Contract.Assert(targets.Count > 0);
foreach (string target in targets)
{
pipDataBuilder.Add(PipDataAtom.FromString($"/t:{target}"));
}
}
else
{
// The prediction for the targets to execute is not available. Just log this as a warning for now, defaults targets will be used.
Tracing.Logger.Log.ProjectIsNotSpecifyingTheProjectReferenceProtocol(
m_context.LoggingContext,
Location.FromFile(project.FullPath.ToString(PathTable)),
project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable));
}
// Pass the output result cache file if present
if (outputResultCacheFile != AbsolutePath.Invalid)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
// Flag /orc is the short form of /outputResultsCache, and part of MSBuild 'build in isolation' mode.
// By specifying this flag, MSBuild will write the build result at the end of this invocation into the cache file
pipDataBuilder.Add(PipDataAtom.FromString("/orc:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(outputResultCacheFile));
}
}
else
{
// In legacy (non-isolated) mode, we still have to rely on SDKs honoring this flag
pipDataBuilder.Add(PipDataAtom.FromString("/p:buildprojectreferences=false"));
}
failureDetail = string.Empty;
return(true);
}