public PipDataTests(ITestOutputHelper output)
: base(output)
{
m_pathTable = new PathTable();
m_expectedStringId0 = StringId.Create(m_pathTable.StringTable, ExpectedString0);
m_pipDataBuilder = new PipDataBuilder(m_pathTable.StringTable);
m_expectedStringId0 = StringId.Create(m_pathTable.StringTable, ExpectedString0);
m_uniqueEntry0 = AbsolutePath.Create(m_pathTable, A("c", "unique to fragment 0"));
// BEGIN ADDING ARGUMENTS
m_cursorStart = m_pipDataBuilder.CreateCursor();
AddStandardBlock(m_pipDataBuilder);
m_cursor0 = m_pipDataBuilder.CreateCursor();
using (m_pipDataBuilder.StartFragment(Escaping0, m_separator0))
{
m_pipDataBuilder.Add(m_uniqueEntry0);
AddStandardBlock(m_pipDataBuilder);
}
m_cursor1 = m_pipDataBuilder.CreateCursor();
using (m_pipDataBuilder.StartFragment(Escaping1, Separator1))
{
AddStandardBlock(m_pipDataBuilder);
m_pipDataBuilder.Add(UniqueEntry1);
}
m_cursorEnd = m_pipDataBuilder.CreateCursor();
// END ADDING ARGUMENTS
}
private static void AddLogArgument(PipDataBuilder pipDataBuilder, int loggerNumber, AbsolutePath logFile, string verbosity)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString(I($"/flp{loggerNumber}:logfile=")));
pipDataBuilder.Add((PipDataAtom.FromAbsolutePath(logFile)));
pipDataBuilder.Add(PipDataAtom.FromString(I($";{verbosity}")));
}
}
private static void AddMsBuildProperty(PipDataBuilder pipDataBuilder, string key, string value)
{
// Make sure properties are always quoted, since MSBuild doesn't handle semicolon separated
// properties without quotes
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString("/p:"));
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString(key));
}
pipDataBuilder.Add(PipDataAtom.FromString("=\""));
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString(value));
}
pipDataBuilder.Add(PipDataAtom.FromString("\""));
}
}
private void AddOption <TValue>(string optionName, TValue value, Action <PipDataBuilder, TValue> writeValue)
{
Contract.Requires(!string.IsNullOrEmpty(optionName));
Contract.Requires(writeValue != null);
Contract.Assert(!m_finished);
using (m_builder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, string.Empty))
{
m_builder.Add(optionName);
writeValue(m_builder, value);
}
}
private void AddOption <TValue>(string prefix, TValue value, bool valueIsEmpty, Action <PipDataBuilder, TValue> writeValue)
{
// prefix and value are null -> skip
if (string.IsNullOrEmpty(prefix) && valueIsEmpty)
{
return;
}
if (string.IsNullOrEmpty(prefix))
{
// This is unnamed argument
writeValue(ArgumentsBuilder, value);
return;
}
if (valueIsEmpty)
{
// This is a flag (or switch) kind of arguments
ArgumentsBuilder.Add(prefix);
return;
}
// both prefix and value are non-empty
// - handle the special case when prefix ends with space
// -> (1) add trimmed prefix, (2) add raw space, (3) add value
if (prefix.EndsWith(" ", StringComparison.OrdinalIgnoreCase))
{
AddRawText(prefix.TrimEnd(' '));
writeValue(ArgumentsBuilder, value);
return;
}
using (ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, EmptyStringId))
{
ArgumentsBuilder.Add(prefix);
writeValue(ArgumentsBuilder, value);
}
}
private void AddJavaScriptArgumentToBuilder(PipDataBuilder argumentsBuilder, JavaScriptArgument value)
{
switch (value.GetValue())
{
case string s:
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, m_context.StringTable.Empty))
{
argumentsBuilder.Add(s);
}
break;
case AbsolutePath absolutePath:
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, m_context.StringTable.Empty))
{
argumentsBuilder.Add(absolutePath);
}
break;
case RelativePath relativePath:
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, m_context.StringTable.Empty))
{
argumentsBuilder.Add(relativePath);
}
break;
case PathAtom pathAtom:
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, m_context.StringTable.Empty))
{
argumentsBuilder.Add(pathAtom);
}
break;
default:
Contract.Assert(false, $"Unexpected argument '{value.GetType()}'");
break;
}
}
public static void Add(this PipDataBuilder @this, string prefix, RelativePath value)
{
var escaping = PipDataFragmentEscaping.CRuntimeArgumentRules;
var separator = System.IO.Path.DirectorySeparatorChar.ToString();
using (@this.StartFragment(escaping, separator))
{
if (!string.IsNullOrEmpty(prefix))
{
@this.Add(prefix);
}
foreach (var atom in value.GetAtoms())
{
@this.Add(atom);
}
}
}
private static Process CreateConsoleProcessInContainer(
BuildXLContext context,
TempFileStorage tempFiles,
PathTable pt,
string arguments,
ReadOnlyArray <FileArtifactWithAttributes> outputFiles,
ReadOnlyArray <DirectoryArtifact> directoryOutputs,
ContainerIsolationLevel containerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs)
{
var executableFileArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(context.PathTable, CmdHelper.CmdX64));
var argumentBuilder = new PipDataBuilder(context.PathTable.StringTable);
argumentBuilder.Add("/d");
argumentBuilder.Add("/c");
using (argumentBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
foreach (var arg in arguments.Split(new[] { ' ' }, System.StringSplitOptions.RemoveEmptyEntries))
{
argumentBuilder.Add(arg);
}
}
string workingDirectory = tempFiles.GetUniqueDirectory();
var workingDirectoryAbsolutePath = AbsolutePath.Create(context.PathTable, workingDirectory);
string uniqueOutputDirectory = tempFiles.GetUniqueDirectory();
var uniqueOutputDirectoryPath = AbsolutePath.Create(context.PathTable, uniqueOutputDirectory);
string uniqueRedirectedOutputDirectory = tempFiles.GetUniqueDirectory("redirected");
var uniqueRedirectedOutputDirectoryPath = AbsolutePath.Create(context.PathTable, uniqueRedirectedOutputDirectory);
var pip = new Process(
executableFileArtifact,
workingDirectoryAbsolutePath,
argumentBuilder.ToPipData(" ", PipDataFragmentEscaping.NoEscaping),
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .FromWithoutCopy(),
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pt),
null,
null,
dependencies: ReadOnlyArray <FileArtifact> .FromWithoutCopy(new[] { executableFileArtifact }),
outputs: outputFiles,
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty,
directoryOutputs: directoryOutputs,
orderDependencies: ReadOnlyArray <PipId> .Empty,
untrackedPaths: ReadOnlyArray <AbsolutePath> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .Empty,
tags: ReadOnlyArray <StringId> .Empty,
successExitCodes: ReadOnlyArray <int> .Empty,
semaphores: ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty,
options: Process.Options.NeedsToRunInContainer,
uniqueOutputDirectory: uniqueOutputDirectoryPath,
uniqueRedirectedDirectoryRoot: uniqueRedirectedOutputDirectoryPath,
containerIsolationLevel: containerIsolationLevel);
return(pip);
}
private bool TryAddMsBuildArguments(ProjectWithPredictions project, PipDataBuilder pipDataBuilder, AbsolutePath logDirectory, AbsolutePath outputResultCacheFile, out string failureDetail)
{
// Common arguments to all MsBuildExe invocations
pipDataBuilder.AddRange(s_commonArgumentsToMsBuildExe.Select(argument => PipDataAtom.FromString(argument)));
// Log verbosity
if (!TryGetLogVerbosity(m_resolverSettings.LogVerbosity, out string logVerbosity))
{
failureDetail = $"Cannot set the MSBuild log verbosity. '{m_resolverSettings.LogVerbosity}' is not a valid option.";
return(false);
}
AddLogArgument(pipDataBuilder, 1, logDirectory.Combine(PathTable, "msbuild.log"), $"Verbosity={logVerbosity}");
AddLogArgument(pipDataBuilder, 2, logDirectory.Combine(PathTable, "msbuild.wrn"), "Verbosity=Quiet;warningsonly");
AddLogArgument(pipDataBuilder, 3, logDirectory.Combine(PathTable, "msbuild.err"), "Verbosity=Quiet;errorsonly");
AddLogArgument(pipDataBuilder, 4, logDirectory.Combine(PathTable, "msbuild.prf"), "PerformanceSummary");
// Global properties on the project are turned into build parameters
foreach (var kvp in project.GlobalProperties)
{
AddMsBuildProperty(pipDataBuilder, kvp.Key, kvp.Value);
}
// Configure binary logger if specified
if (m_resolverSettings.EnableBinLogTracing == true)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString("/binaryLogger:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(logDirectory.Combine(PathTable, "msbuild.binlog")));
}
}
// Targets to execute.
var targets = project.PredictedTargetsToExecute.Targets;
Contract.Assert(targets.Count > 0);
foreach (string target in targets)
{
pipDataBuilder.Add(PipDataAtom.FromString($"/t:{target}"));
}
// Pass the output result cache file if present
if (outputResultCacheFile != AbsolutePath.Invalid)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
// Flag /orc is the short form of /outputResultsCache, and part of MSBuild 'build in isolation' mode.
// By specifying this flag, MSBuild will write the build result at the end of this invocation into the cache file
pipDataBuilder.Add(PipDataAtom.FromString("/orc:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(outputResultCacheFile));
}
}
else
{
// In legacy (non-isolated) mode, we still have to rely on SDKs honoring this flag
pipDataBuilder.Add(PipDataAtom.FromString("/p:buildprojectreferences=false"));
}
failureDetail = string.Empty;
return(true);
}
private bool TryAddMsBuildArguments(ProjectWithPredictions project, PipDataBuilder pipDataBuilder, AbsolutePath logDirectory, AbsolutePath outputResultCacheFile, out string failureDetail)
{
// Common arguments to all MsBuildExe invocations
pipDataBuilder.AddRange(s_commonArgumentsToMsBuildExe.Select(argument => PipDataAtom.FromString(argument)));
// Log verbosity
if (!TryGetLogVerbosity(m_resolverSettings.LogVerbosity, out string logVerbosity))
{
failureDetail = $"Cannot set the MSBuild log verbosity. '{m_resolverSettings.LogVerbosity}' is not a valid option.";
return(false);
}
AddLogArgument(pipDataBuilder, 1, logDirectory.Combine(PathTable, "msbuild.log"), $"Verbosity={logVerbosity}");
AddLogArgument(pipDataBuilder, 2, logDirectory.Combine(PathTable, "msbuild.wrn"), "Verbosity=Quiet;warningsonly");
AddLogArgument(pipDataBuilder, 3, logDirectory.Combine(PathTable, "msbuild.err"), "Verbosity=Quiet;errorsonly");
AddLogArgument(pipDataBuilder, 4, logDirectory.Combine(PathTable, "msbuild.prf"), "PerformanceSummary");
// Global properties on the project are turned into build parameters
foreach (var kvp in project.GlobalProperties)
{
AddMsBuildProperty(pipDataBuilder, kvp.Key, kvp.Value);
}
// Configure binary logger if specified
if (m_resolverSettings.EnableBinLogTracing == true)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString("/binaryLogger:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(logDirectory.Combine(PathTable, "msbuild.binlog")));
}
}
// Targets to execute.
// If the prediction is available, there should be at least one target (otherwise it makes no sense to schedule the project, and we should have caught this earlier)
// If the prediction is not available, we fallback to call default targets (which means not passing any specific /t:). A more strict policy would be to bail out
// here saying that the project is not complying to the target protocol specification. We leave it relaxed for now, but we log it.
// https://github.com/Microsoft/msbuild/blob/master/documentation/specs/static-graph.md
if (project.PredictedTargetsToExecute.IsPredictionAvailable)
{
var targets = project.PredictedTargetsToExecute.Targets;
Contract.Assert(targets.Count > 0);
foreach (string target in targets)
{
pipDataBuilder.Add(PipDataAtom.FromString($"/t:{target}"));
}
}
else
{
// The prediction for the targets to execute is not available. Just log this as a warning for now, defaults targets will be used.
Tracing.Logger.Log.ProjectIsNotSpecifyingTheProjectReferenceProtocol(
m_context.LoggingContext,
Location.FromFile(project.FullPath.ToString(PathTable)),
project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable));
}
// Pass the output result cache file if present
if (outputResultCacheFile != AbsolutePath.Invalid)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
// Flag /orc is the short form of /outputResultsCache, and part of MSBuild 'build in isolation' mode.
// By specifying this flag, MSBuild will write the build result at the end of this invocation into the cache file
pipDataBuilder.Add(PipDataAtom.FromString("/orc:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(outputResultCacheFile));
}
}
else
{
// In legacy (non-isolated) mode, we still have to rely on SDKs honoring this flag
pipDataBuilder.Add(PipDataAtom.FromString("/p:buildprojectreferences=false"));
}
failureDetail = string.Empty;
return(true);
}
/// <summary>
/// Creates arguments.
/// </summary>
internal static PipData CreateCmdArguments(
StringTable stringTable,
IEnumerable <FileArtifact> dependencies,
IEnumerable <FileArtifact> outputs,
bool includeWarning = false)
{
Contract.Requires(dependencies != null, "Argument dependencies cannot be null");
Contract.Requires(outputs != null, "Argument outputs cannot be null");
var pipDataBuilder = new PipDataBuilder(stringTable);
int i = 0;
pipDataBuilder.Add("/d");
pipDataBuilder.Add("/c");
foreach (FileArtifact output in outputs)
{
if (i > 0)
{
pipDataBuilder.Add("&");
}
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
var allDependencies = dependencies.ToList();
pipDataBuilder.Add("type");
foreach (FileArtifact dependency in allDependencies)
{
pipDataBuilder.Add(dependency);
}
pipDataBuilder.Add(">");
pipDataBuilder.Add(output);
}
pipDataBuilder.Add("&");
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
pipDataBuilder.Add("echo");
pipDataBuilder.Add("buildxl");
pipDataBuilder.Add(">>");
pipDataBuilder.Add(output);
}
i++;
}
if (includeWarning)
{
if (outputs.Any())
{
pipDataBuilder.Add("&&");
}
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
pipDataBuilder.Add("echo");
pipDataBuilder.Add(WarningRegexDescription);
}
}
return(pipDataBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules));
}
private bool TryAddMsBuildArguments(ProjectWithPredictions <AbsolutePath> project, Qualifier qualifier, PipDataBuilder pipDataBuilder, AbsolutePath logDirectory, out string failureDetail)
{
// Common arguments to all MsBuildExe invocations
pipDataBuilder.AddRange(s_commonArgumentsToMsBuildExe.Select(argument => PipDataAtom.FromString(argument)));
// Log verbosity
if (!TryGetLogVerbosity(m_resolverSettings.LogVerbosity, out string logVerbosity))
{
failureDetail = $"Cannot set the MSBuild log verbosity. '{m_resolverSettings.LogVerbosity}' is not a valid option.";
return(false);
}
AddLogArgument(pipDataBuilder, 1, logDirectory.Combine(PathTable, "msbuild.log"), $"Verbosity={logVerbosity}");
AddLogArgument(pipDataBuilder, 2, logDirectory.Combine(PathTable, "msbuild.wrn"), "Verbosity=Quiet;warningsonly");
AddLogArgument(pipDataBuilder, 3, logDirectory.Combine(PathTable, "msbuild.err"), "Verbosity=Quiet;errorsonly");
AddLogArgument(pipDataBuilder, 4, logDirectory.Combine(PathTable, "msbuild.prf"), "PerformanceSummary");
// Global properties on the project are turned into build parameters
foreach (var kvp in project.GlobalProperties)
{
AddMsBuildProperty(pipDataBuilder, kvp.Key, kvp.Value);
}
// The specified qualifier, unless overridden by a global property, is turned into a property as well
// This means that:
// 1) if the project is not being referenced with a specific property that matters to the qualifier, the requested qualifier is used
// 2) if a particular property (e.g. platform) is set when referencing the project, that is honored
foreach (StringId key in qualifier.Keys)
{
string keyAsString = key.ToString(m_context.StringTable);
if (!project.GlobalProperties.ContainsKey(keyAsString))
{
var success = qualifier.TryGetValue(m_context.StringTable, keyAsString, out string value);
Contract.Assert(success);
AddMsBuildProperty(pipDataBuilder, keyAsString, value);
}
}
// Configure binary logger if specified
if (m_resolverSettings?.EnableBinLogTracing == true)
{
using (pipDataBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
pipDataBuilder.Add(PipDataAtom.FromString("/binaryLogger:"));
pipDataBuilder.Add(PipDataAtom.FromAbsolutePath(logDirectory.Combine(PathTable, "msbuild.binlog")));
}
}
// Targets to execute.
// If the prediction is available, there should be at least one target (otherwise it makes no sense to schedule the project, and we should have caught this earlier)
// If the prediction is not available, we fallback to call default targets (which means not passing any specific /t:). A more strict policy would be to bail out
// here saying that the project is not complying to the target protocol specification. We leave it relaxed for now, but we log it.
// https://github.com/Microsoft/msbuild/blob/master/documentation/specs/static-graph.md
if (project.PredictedTargetsToExecute.IsPredictionAvailable)
{
var targets = project.PredictedTargetsToExecute.Targets;
Contract.Assert(targets.Count > 0);
foreach (string target in targets)
{
pipDataBuilder.Add(PipDataAtom.FromString($"/t:{target}"));
}
}
else
{
// The prediction for the targets to execute is not available. Just log this as a warning for now, defaults targets will be used.
Tracing.Logger.Log.ProjectIsNotSpecifyingTheProjectReferenceProtocol(
m_context.LoggingContext,
Location.FromFile(project.FullPath.ToString(PathTable)),
project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable));
}
failureDetail = string.Empty;
return(true);
}