/// <summary>
/// Renders <see cref="PipData"/> to string.
/// </summary>
public string RenderPipData(PipData pipData)
{
var rootExpander = new RootExpander(PathTable);
Func <AbsolutePath, string> expandRoot = absPath => PathTable.ExpandName(absPath.Value, rootExpander);
return(pipData.ToString(expandRoot, PathTable.StringTable, PipData.MaxMonikerRenderer));
}
/// <summary>
/// Schedules an IPC pip.
/// </summary>
public IpcPip ScheduleIpcPip(
IIpcMoniker moniker,
PipId?servicePipId,
ProcessBuilder ipcProcessBuilder,
FileArtifact outputFile,
bool isServiceFinalization,
PipConstructionHelper helper = null)
{
var ipcClientInfo = new IpcClientInfo(StringId.Create(Context.StringTable, moniker.Id), new ClientConfig(0, 0));
PipData arguments = ipcProcessBuilder.ArgumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
ReadOnlyArray <FileArtifact> fileDependencies = ipcProcessBuilder.GetInputFilesSoFar();
if (!(helper ?? m_defaultConstructionHelper).TryAddIpc(
ipcClientInfo,
arguments,
outputFile,
servicePipDependencies: servicePipId != null ? ReadOnlyArray <PipId> .From(new[] { servicePipId.Value }) : ReadOnlyArray <PipId> .Empty,
fileDependencies: fileDependencies,
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty,
skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .Empty,
isServiceFinalization: isServiceFinalization,
mustRunOnMaster: false,
tags: new string[0],
out IpcPip ipcPip))
{
throw new BuildXLTestException("Failed to add ipc pip");
}
return(ipcPip);
}
/// <nodoc />
public void SetEnvironmentVariable(StringId key, PipData value)
{
Contract.Requires(key.IsValid);
Contract.Requires(value.IsValid);
m_environmentVariables[key] = value;
}
/// <summary>
/// Verifies the resulting arguments after full graph construction.
/// </summary>
/// <param name="graph">Resulting graph.</param>
/// <param name="fragmentOrigin">Graph fragment where the arguments are constructed.</param>
/// <param name="outputInFragmentOrigin">Output file to identify pip.</param>
/// <param name="expectedArguments">Expected arguments.</param>
private void VerifyResultingArguments(PipGraph graph, TestPipGraphFragment fragmentOrigin, FileArtifact outputInFragmentOrigin, PipData expectedArguments)
{
var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(RemapFragmentPath(fragmentOrigin, outputInFragmentOrigin)));
XAssert.IsTrue(pipId.IsValid);
Pip pip = graph.PipTable.HydratePip(pipId, PipQueryContext.PipGraphGetProducingPip);
XAssert.IsNotNull(pip);
PipData actualArguments = PipData.Invalid;
if (pip is Process process)
{
actualArguments = process.Arguments;
}
else if (pip is IpcPip ipcPip)
{
actualArguments = ipcPip.MessageBody;
}
else
{
XAssert.Fail("No arguments associated with pip");
}
string expected = expectedArguments.ToString(Context.PathTable).ToUpperInvariant();
string actual = actualArguments.ToString(Context.PathTable).ToUpperInvariant();
XAssert.AreEqual(expected, actual);
}
/// <nodoc />
public bool TryAddIpc(
IpcClientInfo ipcClientInfo,
PipData arguments,
FileArtifact outputFile,
ReadOnlyArray <PipId> servicePipDependencies,
ReadOnlyArray <FileArtifact> fileDependencies,
ReadOnlyArray <DirectoryArtifact> directoryDependencies,
ReadOnlyArray <FileOrDirectoryArtifact> skipMaterializationFor,
bool isServiceFinalization,
bool mustRunOnMaster,
string[] tags,
out IpcPip ipcPip)
{
ipcPip = new IpcPip(
ipcClientInfo,
arguments,
outputFile: outputFile,
servicePipDependencies: servicePipDependencies,
fileDependencies: fileDependencies,
directoryDependencies: directoryDependencies,
skipMaterializationFor: skipMaterializationFor,
isServiceFinalization: isServiceFinalization,
mustRunOnMaster: mustRunOnMaster,
tags: ToStringIds(tags),
provenance: CreatePipProvenance(string.Empty)
);
if (PipGraph != null)
{
var success = PipGraph.AddIpcPip(ipcPip, GetValuePipId());
return(success);
}
return(true);
}
/// <nodoc />
public bool TryWriteFile(
AbsolutePath destination,
PipData content,
WriteFileEncoding encoding,
string[] tags,
string description,
out FileArtifact fileArtifact)
{
Contract.Requires(destination.IsValid);
Contract.Requires(content.IsValid);
fileArtifact = FileArtifact.CreateSourceFile(destination).CreateNextWrittenVersion();
var pip = new WriteFile(
fileArtifact,
content,
encoding,
ToStringIds(tags),
CreatePipProvenance(description));
if (PipGraph != null)
{
return(PipGraph.AddWriteFile(pip, GetValuePipId()));
}
return(true);
}
public ProcessBuilder WithArguments(PipData arguments)
{
Contract.Assert(m_argumentsFactory == null, "Only WithArguments or WithArgumentsFactory method could be called, but not both.");
m_arguments = arguments;
return(this);
}
private IExpression Generate(PipData pipData)
{
return(new ObjectLiteralExpression(
new PropertyAssignment("escaping", new LiteralExpression(pipData.FragmentEscaping.ToString())),
new PropertyAssignment("separator", Generate(pipData.FragmentSeparator)),
new PropertyAssignment("items", Generate(pipData.ToList(), Generate))));
}
protected static StringId ExtractMonikerValueFromPipData(PipData arguments)
{
XAssert.AreEqual(1, arguments.FragmentCount, "expected 1 fragment");
PipFragment fragment = arguments.First();
XAssert.AreEqual(PipFragmentType.IpcMoniker, fragment.FragmentType);
return(fragment.GetIpcMonikerValue());
}
internal ResponseFileSpecification(bool forceCreation, Cursor firstArg, bool requiresArgument, string prefix, AbsolutePath explicitPath, PipData explicitData)
{
m_forceCreation = forceCreation;
m_firstArg = firstArg;
m_requiresArgument = requiresArgument;
m_prefix = prefix;
m_explicitPath = explicitPath;
m_explicitData = explicitData;
}
private void VerifyFullPipData(PipData pipData)
{
var enumerator = pipData.GetEnumerator();
XAssert.AreEqual(NumStandardBlockFragments + 2, pipData.FragmentCount);
VerifyStandardBlock(ref enumerator);
VerifyPipDataFromCursor0(enumerator);
}
private PipData FinishArgumentsAndOverflowArgumentsToResponseFile(DirectoryArtifact defaultDirectory)
{
Contract.Requires(defaultDirectory.IsValid);
PipData arguments = default;
var argumentsBuilder = m_argumentsBuilder.Instance;
if (!m_responseFileFirstArg.IsDefault)
{
bool mkRespFile = m_responseFileForceCreation;
if (!mkRespFile)
{
// make a response file only if the command-line is too long
arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
// Normalize choice to use response file by assuming paths are of length max path with a space. This will
// ensure there are no cases where changing the root will change whether a response file is used.
int cmdLineLength = arguments.GetMaxPossibleLength(m_pathTable.StringTable);
mkRespFile = cmdLineLength > MaxCommandLineLength;
}
if (mkRespFile)
{
// create a pip data for the stuff in the response file
ResponseFileData = argumentsBuilder.ToPipData("\r\n", PipDataFragmentEscaping.CRuntimeArgumentRules, m_responseFileFirstArg);
// generate the file
ResponseFile = FileArtifact.CreateSourceFile(defaultDirectory.Path.Combine(m_pathTable, PathAtom.Create(m_pathTable.StringTable, "args.rsp")));
argumentsBuilder.TrimEnd(m_responseFileFirstArg);
AddUntrackedFile(ResponseFile);
if (string.IsNullOrEmpty(m_responseFilePrefix))
{
argumentsBuilder.Add(ResponseFile);
}
else
{
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, m_pathTable.StringTable.Empty))
{
argumentsBuilder.Add(m_responseFilePrefix);
argumentsBuilder.Add(ResponseFile);
}
}
arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
}
}
else
{
arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
}
return(arguments);
}
private void IterateClArguments(Project project, PipData arguments, bool isNested = false)
{
Action <object> action = null;
var commandLine = arguments.ToString(Context.PathTable);
foreach (var arg in arguments)
{
var type = arg.FragmentType;
if (action != null)
{
action(GetObjectValue(arg));
action = null;
}
else if (type == PipFragmentType.AbsolutePath && !isNested)
{
var path = GetPathValue(arg);
AddSourceItem(path, project, "ClCompile");
}
else if (type == PipFragmentType.StringLiteral)
{
var strValue = arg.GetStringIdValue().ToString(Context.StringTable);
if (strValue == "/I")
{
action = (obj) =>
{
if ((AbsolutePath)obj != SpecDirectory)
{
IncludeDirsByQualifier.Add(project.FriendlyQualifier, (AbsolutePath)obj);
}
};
}
else if (strValue == "/D")
{
action = (obj) =>
{
if (obj is PipData)
{
obj = ((PipData)obj).ToString(Context.PathTable);
}
else if (!(obj is string))
{
Contract.Assert(false, "Expecting string or PipData as a preprocessor definition argument");
}
ConstantsByQualifier.Add(project.FriendlyQualifier, (string)obj);
};
}
}
else if (type == PipFragmentType.NestedFragment)
{
IterateClArguments(project, arg.GetNestedFragmentValue(), true);
}
}
}
private void IterateLinkArguments(Project project, PipData arguments, bool isNested = false)
{
foreach (var arg in arguments)
{
var type = arg.FragmentType;
if (type == PipFragmentType.AbsolutePath && !isNested)
{
// Sources
var path = GetPathValue(arg);
project.RawReferences.Add(path);
}
}
}
internal PipProvenance CreatePipProvenance(PipData usage)
{
var result = new PipProvenance(
GetNextSemiStableHash(),
moduleId: m_moduleId,
moduleName: StringId.Create(Context.StringTable, m_moduleName),
outputValueSymbol: m_valuePip.Symbol,
token: m_valuePip.LocationData,
qualifierId: m_valuePip.Qualifier,
usage: usage);
return(result);
}
/// <summary>
/// Adds a fake write file pip that produces to the given destination path.
/// </summary>
public WriteFile AddWriteFilePip(AbsolutePath destinationPath)
{
Contract.Requires(destinationPath != null);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationPath).CreateNextWrittenVersion();
PipData contents = PipDataBuilder.CreatePipData(m_context.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "content");
var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(m_context));
writeFile.PipId = AllocateNextPipId();
m_pips.Add(writeFile.PipId, writeFile);
m_pathProducers.Add(destinationArtifact, writeFile);
return(writeFile);
}
private void VerifyNested1(PipFragment fragment)
{
PipData nestedData = fragment.GetNestedFragmentValue();
XAssert.AreEqual(NumStandardBlockFragments + 1, nestedData.FragmentCount);
using (var nestedEnumerator = nestedData.GetEnumerator())
{
var localEnumerator = nestedEnumerator;
VerifyStandardBlock(ref localEnumerator);
AssertMoveNext(ref localEnumerator, out fragment, PipFragmentType.StringLiteral);
XAssert.AreEqual(UniqueEntry1, m_pathTable.StringTable.GetString(fragment.GetStringIdValue()));
XAssert.IsFalse(localEnumerator.MoveNext());
}
}
private void VerifyNested0(PipFragment fragment)
{
PipData nestedData = fragment.GetNestedFragmentValue();
XAssert.AreEqual(NumStandardBlockFragments + 1, nestedData.FragmentCount);
using (var nestedEnumerator = nestedData.GetEnumerator())
{
var localEnumerator = nestedEnumerator;
AssertMoveNext(ref localEnumerator, out fragment, PipFragmentType.AbsolutePath);
XAssert.AreEqual(m_uniqueEntry0, fragment.GetPathValue());
VerifyStandardBlock(ref localEnumerator);
XAssert.IsFalse(localEnumerator.MoveNext());
}
}
/// <nodoc />
public bool TrySealDirectory(
AbsolutePath directoryRoot,
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> contents,
SortedReadOnlyArray <DirectoryArtifact, OrdinalDirectoryArtifactComparer> outputDirectorycontents,
SealDirectoryKind kind,
string[] tags,
string description,
string[] patterns,
out DirectoryArtifact sealedDirectory,
bool scrub = false)
{
Contract.Requires(directoryRoot.IsValid);
Contract.Requires(contents.IsValid);
Contract.Requires(outputDirectorycontents.IsValid);
PipData usage = PipDataBuilder.CreatePipData(Context.StringTable, string.Empty, PipDataFragmentEscaping.NoEscaping, description != null
? new PipDataAtom[] { description }
: new PipDataAtom[] { "'", directoryRoot, "' [", contents.Length.ToString(CultureInfo.InvariantCulture), " files - ",
outputDirectorycontents.Length.ToString(CultureInfo.InvariantCulture), " output directories]" });
var pip = new SealDirectory(
directoryRoot,
contents,
outputDirectorycontents,
kind,
CreatePipProvenance(usage),
ToStringIds(tags),
ToStringIds(patterns),
scrub);
if (PipGraph != null)
{
sealedDirectory = PipGraph.AddSealDirectory(pip, GetValuePipId());
if (!sealedDirectory.IsValid)
{
return(false);
}
}
else
{
sealedDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(directoryRoot);
}
return(true);
}
/// <nodoc/>
public bool TryComposeSharedOpaqueDirectory(
AbsolutePath directoryRoot,
IReadOnlyList <DirectoryArtifact> contents,
SealDirectoryContentFilter?contentFilter,
[CanBeNull] string description,
[CanBeNull] string[] tags,
out DirectoryArtifact sharedOpaqueDirectory)
{
Contract.Requires(directoryRoot.IsValid);
Contract.Requires(contents != null);
if (PipGraph == null)
{
sharedOpaqueDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(directoryRoot);
return(true);
}
PipData usage = PipDataBuilder.CreatePipData(Context.StringTable, string.Empty, PipDataFragmentEscaping.NoEscaping, description != null
? new PipDataAtom[] { description }
: new PipDataAtom[] { "'", directoryRoot, "' [", contents.Count.ToString(CultureInfo.InvariantCulture),
" shared opaque directories, filter: ",
contentFilter.HasValue ? $"'{contentFilter.Value.Regex}' (kind: {Enum.GetName(typeof(SealDirectoryContentFilter.ContentFilterKind), contentFilter.Value.Kind)})" : "''", "]" });
sharedOpaqueDirectory = PipGraph.ReserveSharedOpaqueDirectory(directoryRoot);
var pip = new CompositeSharedOpaqueSealDirectory(
directoryRoot,
contents,
CreatePipProvenance(usage),
ToStringIds(tags),
contentFilter);
// The seal directory is ready to be initialized, since the directory artifact has been reserved already
pip.SetDirectoryArtifact(sharedOpaqueDirectory);
sharedOpaqueDirectory = PipGraph.AddSealDirectory(pip, GetValuePipId());
if (!sharedOpaqueDirectory.IsValid)
{
return(false);
}
return(true);
}
internal void ProcessNestedArgument(PipData nested, StringBuilder builder)
{
foreach (var arg in nested)
{
switch (arg.FragmentType)
{
case PipFragmentType.StringLiteral:
builder.AppendLine(arg.GetStringIdValue().ToString(Context.StringTable));
break;
case PipFragmentType.AbsolutePath:
builder.AppendLine(arg.GetPathValue().ToString(Context.PathTable));
break;
case PipFragmentType.NestedFragment:
builder.AppendLine();
builder.AppendLine();
ProcessNestedArgument(arg.GetNestedFragmentValue(), builder);
break;
}
}
}
/// <summary>
/// Gets the command line arguments for the process.
/// </summary>
public PipData GetArgumentsDataFromProcess(Process process)
{
PipData arguments = process.Arguments;
if (process.ResponseFile.IsValid)
{
var responseFileData = process.ResponseFileData;
PipDataBuilder pipDataBuilder = new PipDataBuilder(StringTable);
// Add all the arguments from the command line excluding the response file (the last fragment)
foreach (var fragment in process.Arguments.Take(process.Arguments.FragmentCount - 1).Concat(responseFileData))
{
Contract.Assume(fragment.FragmentType != PipFragmentType.Invalid);
pipDataBuilder.Add(fragment);
}
arguments = pipDataBuilder.ToPipData(arguments.FragmentSeparator, arguments.FragmentEscaping);
}
return(arguments);
}
private static IEnumerable <string> ConvertArgumentsToStringArray(PipData pipData, PathTable pathTable)
{
var result = new List <string>();
foreach (PipFragment fragment in pipData)
{
Contract.Assume(fragment.FragmentType != PipFragmentType.Invalid);
string s = string.Empty;
switch (fragment.FragmentType)
{
case PipFragmentType.StringLiteral:
s = pathTable.StringTable.GetString(fragment.GetStringIdValue());
break;
case PipFragmentType.AbsolutePath:
s = fragment.GetPathValue().ToString(pathTable);
break;
case PipFragmentType.NestedFragment:
s = fragment.GetNestedFragmentValue().ToString(pathTable);
break;
default:
Contract.Assert(false, "Unhandled fragment type");
break;
}
if (pipData.FragmentEscaping == PipDataFragmentEscaping.CRuntimeArgumentRules)
{
s = CommandLineEscaping.EscapeAsCommandLineWord(s);
}
s += pathTable.StringTable.GetString(pipData.FragmentSeparator);
result.Add(s);
}
return(result);
}
public void PipDataReadWrite()
{
var pipData = m_pipDataBuilder.ToPipData(EnclosingSeparator, EnclosingEscaping);
byte[] bytes = new byte[10000];
int startIndex = 23;
int index = startIndex;
foreach (var entry in pipData.Entries)
{
var expectedIndex = index + PipDataEntry.BinarySize;
entry.Write(bytes, ref index);
// Ensure the correct number of bytes are written
Assert.Equal(expectedIndex, index);
}
var endIndex = index;
index = startIndex;
List <PipDataEntry> readEntries = new List <PipDataEntry>();
while (index < endIndex)
{
var expectedIndex = index + PipDataEntry.BinarySize;
readEntries.Add(PipDataEntry.Read(bytes, ref index));
// Ensure the correct number of bytes are read
Assert.Equal(expectedIndex, index);
}
var readPipData = PipData.CreateInternal(pipData.HeaderEntry, PipDataEntryList.FromEntries(readEntries), StringId.Invalid);
VerifyFullPipData(readPipData);
}
private bool TryConfigureProcessBuilder(ProcessBuilder processBuilder, NinjaNode node, QualifierId qualifierId)
{
SeparateExecutableFromCommands(node.Command, out string executable, out string args);
if (!TryFindExecutablePath(executable, out AbsolutePath exePath))
{
Tracing.Logger.Log.InvalidExecutablePath(m_context.LoggingContext, Location.FromFile(m_specPath.ToString(m_context.PathTable)), node.Command);
return(false);
}
FileArtifact prExeArtifact = FileArtifact.CreateSourceFile(exePath);
processBuilder.Executable = prExeArtifact;
processBuilder.AddInputFile(prExeArtifact);
using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder())
{
var pipDataBuilder = pipDataBuilderWrapper.Instance;
pipDataBuilder.Add(args);
processBuilder.ArgumentsBuilder.Add(pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
}
if (node.ResponseFile.HasValue)
{
using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder())
{
var pipDataBuilder = pipDataBuilderWrapper.Instance;
pipDataBuilder.Add(node.ResponseFile?.Content);
PipData responseFileData = pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping);
// We tell the process builder to write the response file but to not add any arguments to the process (requiresExplicitArgument = false)
// because that information is already in node.Command
var rspFileSpec = ResponseFileSpecification.Builder()
.ExplicitData(responseFileData)
.RequiresArgument(false)
.ExplicitPath((AbsolutePath)node.ResponseFile?.Path)
.Build();
processBuilder.SetResponseFileSpecification(rspFileSpec);
}
}
// TODO: Maybe a better description. Add ninja description or change command for input/outputs
processBuilder.ToolDescription = StringId.Create(m_context.StringTable,
I($"{m_moduleDefinition.Descriptor.Name} - {node.Rule} - {executable} :: [{node.Command}]"));
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads | Process.Options.OutputsMustRemainWritable | Process.Options.OutputsMustRemainWritable;
processBuilder.EnableTempDirectory();
// Working directory - the directory containing the ninja spec file
// Ninja generators may express paths relative to this
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(m_specPath.GetParent(m_context.PathTable));
// Untrack directories
UntrackFilesAndDirectories(processBuilder);
// Allow some surviving child process
AddRequiredSurvivingChildren(processBuilder);
// Environment variables
SetEnvironmentVariables(processBuilder, node);
return(true);
}
private static string CreateString(PipData value, PathTable pathTable)
{
return(value.IsValid ? value.ToString(pathTable) : null);
}
public XElement CreateRow(string key, PipData value)
{
return(value.IsValid ? CreateRow(key, value.ToString(m_pathTable)) : null);
}
/// <summary>
/// Configure the processBuilder's ResponseFile and ResponseFileData according to this specification.
/// This method mutates the processBuilder, changing its argumentsBuilder, ResponseFile and ResponseFileData accordingly.
/// We return the arguments to be passed to the process.
/// </summary>
internal PipData SplitArgumentsAndCreateResponseFileIfNeeded(ProcessBuilder processBuilder, DirectoryArtifact defaultDirectory, PathTable pathTable)
{
PipData arguments = default; // We'll return the actual arguments to be passed to the process.
var argumentsBuilder = processBuilder.ArgumentsBuilder;
var cutoffArg = m_firstArg;
// We will create a response file in the following cases:
// 1. If an explicit response file content was specified, either by having m_explicitData or
// by having m_forceCreation = true
// 2. If the argument line is too long (longer than MaxCommandLineLength) and m_firstArg is not the default.
// An additional argument is added to the process specifying the response file location, prefixed
// by m_prefix, unless m_requiresArgument was set to false.
if (!m_firstArg.IsDefault || m_explicitData.IsValid)
{
bool argumentLineTooLong = false;
if (!m_forceCreation)
{
// make a response file only if the command-line is too long
arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
// Normalize choice to use response file by assuming paths are of length max path with a space. This will
// ensure there are no cases where changing the root will change whether a response file is used.
int cmdLineLength = arguments.GetMaxPossibleLength(pathTable.StringTable);
argumentLineTooLong = cmdLineLength > ProcessBuilder.MaxCommandLineLength;
}
if (m_forceCreation || argumentLineTooLong || m_explicitData.IsValid)
{
// add the explicit contents if we have to
if (m_explicitData.IsValid)
{
if (!argumentLineTooLong)
{
// If there was no overflow, we mark 'here' as the starting
// point for the response file before adding the explicit data as an 'argument'.
cutoffArg = argumentsBuilder.CreateCursor();
}
argumentsBuilder.Add(m_explicitData);
}
// create a pip data for the stuff in the response file
processBuilder.ResponseFileData = argumentsBuilder.ToPipData(Environment.NewLine, PipDataFragmentEscaping.CRuntimeArgumentRules, cutoffArg);
// generate the file
processBuilder.ResponseFile = FileArtifact.CreateSourceFile(GetResponseFilePath(defaultDirectory, pathTable));
argumentsBuilder.TrimEnd(cutoffArg);
processBuilder.AddUntrackedFile(processBuilder.ResponseFile);
if (m_requiresArgument)
{
if (string.IsNullOrEmpty(m_prefix))
{
argumentsBuilder.Add(processBuilder.ResponseFile);
}
else
{
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, pathTable.StringTable.Empty))
{
argumentsBuilder.Add(m_prefix);
argumentsBuilder.Add(processBuilder.ResponseFile);
}
}
}
arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
}
}
else
{
arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
}
return(arguments);
}
private void IterateCscArguments(Project project, PipData arguments, bool isNested = false)
{
Action <object> action = null;
foreach (var arg in arguments)
{
var type = arg.FragmentType;
if (action != null)
{
action(GetObjectValue(arg));
action = null;
}
else if (type == PipFragmentType.AbsolutePath && !isNested)
{
var path = GetPathValue(arg);
AddSourceItem(path, project, "Compile");
}
else if (type == PipFragmentType.StringLiteral)
{
var strValue = arg.GetStringIdValue().ToString(Context.StringTable);
switch (strValue)
{
case "/analyzer:":
action = (obj) => project.AddItem("Analyzer", (AbsolutePath)obj);
break;
case "/r:":
case "/link:":
action = (obj) => project.RawReferences.Add((AbsolutePath)obj);
break;
case "/langversion:":
action = (obj) => project.SetProperty("LangVersion", (string)obj);
break;
case "/target:":
action = (obj) => project.SetProperty("OutputType", (string)obj);
break;
case "/keyfile:":
action = (obj) =>
{
project.SetProperty("AssemblyOriginatorKeyFile", (AbsolutePath)obj);
project.SetProperty("SignAssembly", bool.TrueString);
project.SetProperty("DelaySign", bool.FalseString);
};
break;
case "/define:":
action = (obj) => project.SetProperty("DefineConstants", (string)obj);
break;
case "/unsafe":
action = (obj) =>
{
if ((string)obj == "+")
{
project.SetProperty("AllowUnsafeBlocks", bool.TrueString);
}
};
break;
case "/unsafe+":
project.SetProperty("AllowUnsafeBlocks", bool.TrueString);
break;
case "/recurse:":
action = (obj) =>
{
var expandedPathWithWildCard = ((PipData)obj).ToString(Context.PathTable);
var msbuildMatchString = expandedPathWithWildCard.Replace("*", "**");
project.AddItem("Compile", msbuildMatchString);
// if (!Context.CanWriteToSrc)
// {
// // We only need link if the file goes outside of the projects cone.
// // If the item path and the link are the same path. VisualStudio will not render the file in the project tree.
// RelativePath relativePath;
// if (SpecDirectory.TryGetRelative(Context.PathTable, path, out relativePath) && relativePath.GetAtoms().Length > 1)
// {
// item.SetMetadata("Link", relativePath.ToString(Context.StringTable));
// }
// }
};
break;
case "/out:":
action = (obj) =>
{
var assemblyPath = (AbsolutePath)obj;
// TODO: There should be only one assembly name for a csproj file
var outputFileWithoutExt = assemblyPath.GetName(Context.PathTable).RemoveExtension(Context.StringTable);
project.SetProperty("AssemblyName", outputFileWithoutExt);
project.SetProperty("RootNamespace", outputFileWithoutExt);
var outputDir = assemblyPath.GetParent(Context.PathTable);
SetOutputDirectory(project, outputDir, OutputDirectoryType.Build);
};
break;
case "/ruleset:":
action = (obj) =>
{
project.SetProperty("CodeAnalysisRuleSet", (AbsolutePath)obj);
project.SetProperty("RunCodeAnalysis", bool.TrueString);
};
break;
case "/additionalfile:":
action = (obj) => project.AddItem("AdditionalFiles", (AbsolutePath)obj);
break;
case "/features:":
action = obj =>
{
var features = ((PipData)obj).ToString(Context.PathTable);
project.SetProperty("Features", features);
};
break;
case "/nullable":
action = (obj) =>
{
if ((string)obj == "+")
{
project.SetProperty("Nullable", "enable");
}
};
break;
default:
if (strValue.StartsWith("/target:", StringComparison.OrdinalIgnoreCase))
{
// BuildXL XML specific
project.SetProperty("OutputType", strValue.Substring(8));
}
break;
}
}
else if (type == PipFragmentType.NestedFragment)
{
IterateCscArguments(project, arg.GetNestedFragmentValue(), true);
}
}
}
public async Task Stress()
{
const int N = 5;
const int M = N * N;
var context = BuildXLContext.CreateInstanceForTesting();
var loggingContext = CreateLoggingContextForTest();
var pathTable = context.PathTable;
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var config = ConfigHelpers.CreateDefault(pathTable, tempFiles.GetUniqueFileName(), tempFiles);
using (var pipTable = new PipTable(
context.PathTable,
context.SymbolTable,
initialBufferSize: 1024,
maxDegreeOfParallelism: (Environment.ProcessorCount + 2) / 3,
debug: false))
{
var executionEnvironment = new PipQueueTestExecutionEnvironment(
context,
config,
pipTable,
Path.Combine(TestOutputDirectory, "temp"),
TryGetSubstSourceAndTarget(out string substSource, out string substTarget) ? (substSource, substTarget) : default((string, string)?),
GetSandboxConnection());
Func <RunnablePip, Task <PipResult> > taskFactory = async(runnablePip) =>
{
PipResult result;
var operationTracker = new OperationTracker(runnablePip.LoggingContext);
var pip = runnablePip.Pip;
using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, runnablePip.LoggingContext))
{
result = await TestPipExecutor.ExecuteAsync(operationContext, executionEnvironment, pip);
}
executionEnvironment.MarkExecuted(pip);
return(result);
};
string executable = CmdHelper.OsShellExe;
FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
// This is the only file artifact we reference without a producer. Rather than scheduling a hashing pip, let's just invent one (so fingerprinting can succeed).
executionEnvironment.AddWellKnownFile(executableArtifact, WellKnownContentHashes.UntrackedFile);
using (var phase1PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
{
// phase 1: create some files
var baseFileArtifacts = new List <FileArtifact>();
for (int i = 0; i < N; i++)
{
string destination = tempFiles.GetUniqueFileName();
AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
baseFileArtifacts.Add(destinationArtifact);
PipData contents = PipDataBuilder.CreatePipData(
context.StringTable,
" ",
PipDataFragmentEscaping.CRuntimeArgumentRules,
i.ToString(CultureInfo.InvariantCulture));
var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
var pipId = pipTable.Add((uint)(i + 1), writeFile);
var contentHash = ContentHashingUtilities.HashString(contents.ToString(pathTable));
executionEnvironment.AddExpectedWrite(writeFile, destinationArtifact, contentHash);
var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactory, 0);
runnable.Start(new OperationTracker(loggingContext), loggingContext);
runnable.SetDispatcherKind(DispatcherKind.IO);
phase1PipQueue.Enqueue(runnable);
}
phase1PipQueue.SetAsFinalized();
phase1PipQueue.DrainQueues();
await Task.WhenAll(
Enumerable.Range(0, 2).Select(
async range =>
{
using (var phase2PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
{
// phase 2: do some more with those files
var pips = new ConcurrentDictionary <PipId, Tuple <string, int> >();
var checkerTasks = new ConcurrentQueue <Task>();
Action <PipId, Task <PipResult> > callback =
(id, task) =>
{
XAssert.IsTrue(task.Status == TaskStatus.RanToCompletion);
XAssert.IsFalse(task.Result.Status.IndicatesFailure());
Tuple <string, int> t;
if (!pips.TryRemove(id, out t))
{
XAssert.Fail();
}
checkerTasks.Enqueue(
Task.Run(
() =>
{
string actual = File.ReadAllText(t.Item1).Trim();
// TODO: Make this async
XAssert.AreEqual(actual, t.Item2.ToString());
}));
};
var r = new Random(0);
for (int i = 0; i < M; i++)
{
int sourceIndex = r.Next(baseFileArtifacts.Count);
FileArtifact sourceArtifact = baseFileArtifacts[sourceIndex];
string destination = tempFiles.GetUniqueFileName();
AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
Pip pip;
DispatcherKind queueKind;
switch (r.Next(2))
{
case 0:
pip = new CopyFile(sourceArtifact, destinationArtifact, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
queueKind = DispatcherKind.IO;
executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
break;
case 1:
string workingDirectory =
OperatingSystemHelper.IsUnixOS ? "/tmp" :
Environment.GetFolderPath(Environment.SpecialFolder.Windows);
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
var pipData = OperatingSystemHelper.IsUnixOS ?
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "-c", "'", "cp", sourceArtifact, destinationArtifact, "'") :
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "/d", "/c", "copy", "/B", sourceArtifact, destinationArtifact);
queueKind = DispatcherKind.CPU;
pip = new Process(
executableArtifact,
workingDirectoryAbsolutePath,
pipData,
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .Empty,
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pathTable),
null,
null,
ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact, sourceArtifact),
ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(destinationArtifact.WithAttributes()),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencies(pathTable)),
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(pathTable)),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
break;
default:
Contract.Assert(false);
continue;
}
var pipId = pipTable.Add((uint)((range *M) + N + i + 1), pip);
Func <RunnablePip, Task> taskFactoryWithCallback = async(runnablePip) =>
{
var task = taskFactory(runnablePip);
var pipResult = await task;
callback(pipId, task);
};
var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactoryWithCallback, 0);
runnable.Start(new OperationTracker(loggingContext), loggingContext);
runnable.SetDispatcherKind(queueKind);
phase2PipQueue.Enqueue(runnable);
if (!pips.TryAdd(pipId, Tuple.Create(destination, sourceIndex)))
{
Contract.Assert(false);
}
}
phase2PipQueue.SetAsFinalized();
phase2PipQueue.DrainQueues();
XAssert.AreEqual(0, pips.Count);
await Task.WhenAll(checkerTasks);
}
}));
}
}
}
}
