/// <summary> /// Renders <see cref="PipData"/> to string. /// </summary> public string RenderPipData(PipData pipData) { var rootExpander = new RootExpander(PathTable); Func <AbsolutePath, string> expandRoot = absPath => PathTable.ExpandName(absPath.Value, rootExpander); return(pipData.ToString(expandRoot, PathTable.StringTable, PipData.MaxMonikerRenderer)); }
/// <summary> /// Schedules an IPC pip. /// </summary> public IpcPip ScheduleIpcPip( IIpcMoniker moniker, PipId?servicePipId, ProcessBuilder ipcProcessBuilder, FileArtifact outputFile, bool isServiceFinalization, PipConstructionHelper helper = null) { var ipcClientInfo = new IpcClientInfo(StringId.Create(Context.StringTable, moniker.Id), new ClientConfig(0, 0)); PipData arguments = ipcProcessBuilder.ArgumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules); ReadOnlyArray <FileArtifact> fileDependencies = ipcProcessBuilder.GetInputFilesSoFar(); if (!(helper ?? m_defaultConstructionHelper).TryAddIpc( ipcClientInfo, arguments, outputFile, servicePipDependencies: servicePipId != null ? ReadOnlyArray <PipId> .From(new[] { servicePipId.Value }) : ReadOnlyArray <PipId> .Empty, fileDependencies: fileDependencies, directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty, skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .Empty, isServiceFinalization: isServiceFinalization, mustRunOnMaster: false, tags: new string[0], out IpcPip ipcPip)) { throw new BuildXLTestException("Failed to add ipc pip"); } return(ipcPip); }
/// <nodoc /> public void SetEnvironmentVariable(StringId key, PipData value) { Contract.Requires(key.IsValid); Contract.Requires(value.IsValid); m_environmentVariables[key] = value; }
/// <summary> /// Verifies the resulting arguments after full graph construction. /// </summary> /// <param name="graph">Resulting graph.</param> /// <param name="fragmentOrigin">Graph fragment where the arguments are constructed.</param> /// <param name="outputInFragmentOrigin">Output file to identify pip.</param> /// <param name="expectedArguments">Expected arguments.</param> private void VerifyResultingArguments(PipGraph graph, TestPipGraphFragment fragmentOrigin, FileArtifact outputInFragmentOrigin, PipData expectedArguments) { var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(RemapFragmentPath(fragmentOrigin, outputInFragmentOrigin))); XAssert.IsTrue(pipId.IsValid); Pip pip = graph.PipTable.HydratePip(pipId, PipQueryContext.PipGraphGetProducingPip); XAssert.IsNotNull(pip); PipData actualArguments = PipData.Invalid; if (pip is Process process) { actualArguments = process.Arguments; } else if (pip is IpcPip ipcPip) { actualArguments = ipcPip.MessageBody; } else { XAssert.Fail("No arguments associated with pip"); } string expected = expectedArguments.ToString(Context.PathTable).ToUpperInvariant(); string actual = actualArguments.ToString(Context.PathTable).ToUpperInvariant(); XAssert.AreEqual(expected, actual); }
/// <nodoc /> public bool TryAddIpc( IpcClientInfo ipcClientInfo, PipData arguments, FileArtifact outputFile, ReadOnlyArray <PipId> servicePipDependencies, ReadOnlyArray <FileArtifact> fileDependencies, ReadOnlyArray <DirectoryArtifact> directoryDependencies, ReadOnlyArray <FileOrDirectoryArtifact> skipMaterializationFor, bool isServiceFinalization, bool mustRunOnMaster, string[] tags, out IpcPip ipcPip) { ipcPip = new IpcPip( ipcClientInfo, arguments, outputFile: outputFile, servicePipDependencies: servicePipDependencies, fileDependencies: fileDependencies, directoryDependencies: directoryDependencies, skipMaterializationFor: skipMaterializationFor, isServiceFinalization: isServiceFinalization, mustRunOnMaster: mustRunOnMaster, tags: ToStringIds(tags), provenance: CreatePipProvenance(string.Empty) ); if (PipGraph != null) { var success = PipGraph.AddIpcPip(ipcPip, GetValuePipId()); return(success); } return(true); }
/// <nodoc /> public bool TryWriteFile( AbsolutePath destination, PipData content, WriteFileEncoding encoding, string[] tags, string description, out FileArtifact fileArtifact) { Contract.Requires(destination.IsValid); Contract.Requires(content.IsValid); fileArtifact = FileArtifact.CreateSourceFile(destination).CreateNextWrittenVersion(); var pip = new WriteFile( fileArtifact, content, encoding, ToStringIds(tags), CreatePipProvenance(description)); if (PipGraph != null) { return(PipGraph.AddWriteFile(pip, GetValuePipId())); } return(true); }
public ProcessBuilder WithArguments(PipData arguments) { Contract.Assert(m_argumentsFactory == null, "Only WithArguments or WithArgumentsFactory method could be called, but not both."); m_arguments = arguments; return(this); }
private IExpression Generate(PipData pipData) { return(new ObjectLiteralExpression( new PropertyAssignment("escaping", new LiteralExpression(pipData.FragmentEscaping.ToString())), new PropertyAssignment("separator", Generate(pipData.FragmentSeparator)), new PropertyAssignment("items", Generate(pipData.ToList(), Generate)))); }
protected static StringId ExtractMonikerValueFromPipData(PipData arguments) { XAssert.AreEqual(1, arguments.FragmentCount, "expected 1 fragment"); PipFragment fragment = arguments.First(); XAssert.AreEqual(PipFragmentType.IpcMoniker, fragment.FragmentType); return(fragment.GetIpcMonikerValue()); }
internal ResponseFileSpecification(bool forceCreation, Cursor firstArg, bool requiresArgument, string prefix, AbsolutePath explicitPath, PipData explicitData) { m_forceCreation = forceCreation; m_firstArg = firstArg; m_requiresArgument = requiresArgument; m_prefix = prefix; m_explicitPath = explicitPath; m_explicitData = explicitData; }
private void VerifyFullPipData(PipData pipData) { var enumerator = pipData.GetEnumerator(); XAssert.AreEqual(NumStandardBlockFragments + 2, pipData.FragmentCount); VerifyStandardBlock(ref enumerator); VerifyPipDataFromCursor0(enumerator); }
private PipData FinishArgumentsAndOverflowArgumentsToResponseFile(DirectoryArtifact defaultDirectory) { Contract.Requires(defaultDirectory.IsValid); PipData arguments = default; var argumentsBuilder = m_argumentsBuilder.Instance; if (!m_responseFileFirstArg.IsDefault) { bool mkRespFile = m_responseFileForceCreation; if (!mkRespFile) { // make a response file only if the command-line is too long arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules); // Normalize choice to use response file by assuming paths are of length max path with a space. This will // ensure there are no cases where changing the root will change whether a response file is used. int cmdLineLength = arguments.GetMaxPossibleLength(m_pathTable.StringTable); mkRespFile = cmdLineLength > MaxCommandLineLength; } if (mkRespFile) { // create a pip data for the stuff in the response file ResponseFileData = argumentsBuilder.ToPipData("\r\n", PipDataFragmentEscaping.CRuntimeArgumentRules, m_responseFileFirstArg); // generate the file ResponseFile = FileArtifact.CreateSourceFile(defaultDirectory.Path.Combine(m_pathTable, PathAtom.Create(m_pathTable.StringTable, "args.rsp"))); argumentsBuilder.TrimEnd(m_responseFileFirstArg); AddUntrackedFile(ResponseFile); if (string.IsNullOrEmpty(m_responseFilePrefix)) { argumentsBuilder.Add(ResponseFile); } else { using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, m_pathTable.StringTable.Empty)) { argumentsBuilder.Add(m_responseFilePrefix); argumentsBuilder.Add(ResponseFile); } } arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules); } } else { arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules); } return(arguments); }
private void IterateClArguments(Project project, PipData arguments, bool isNested = false) { Action <object> action = null; var commandLine = arguments.ToString(Context.PathTable); foreach (var arg in arguments) { var type = arg.FragmentType; if (action != null) { action(GetObjectValue(arg)); action = null; } else if (type == PipFragmentType.AbsolutePath && !isNested) { var path = GetPathValue(arg); AddSourceItem(path, project, "ClCompile"); } else if (type == PipFragmentType.StringLiteral) { var strValue = arg.GetStringIdValue().ToString(Context.StringTable); if (strValue == "/I") { action = (obj) => { if ((AbsolutePath)obj != SpecDirectory) { IncludeDirsByQualifier.Add(project.FriendlyQualifier, (AbsolutePath)obj); } }; } else if (strValue == "/D") { action = (obj) => { if (obj is PipData) { obj = ((PipData)obj).ToString(Context.PathTable); } else if (!(obj is string)) { Contract.Assert(false, "Expecting string or PipData as a preprocessor definition argument"); } ConstantsByQualifier.Add(project.FriendlyQualifier, (string)obj); }; } } else if (type == PipFragmentType.NestedFragment) { IterateClArguments(project, arg.GetNestedFragmentValue(), true); } } }
private void IterateLinkArguments(Project project, PipData arguments, bool isNested = false) { foreach (var arg in arguments) { var type = arg.FragmentType; if (type == PipFragmentType.AbsolutePath && !isNested) { // Sources var path = GetPathValue(arg); project.RawReferences.Add(path); } } }
internal PipProvenance CreatePipProvenance(PipData usage) { var result = new PipProvenance( GetNextSemiStableHash(), moduleId: m_moduleId, moduleName: StringId.Create(Context.StringTable, m_moduleName), outputValueSymbol: m_valuePip.Symbol, token: m_valuePip.LocationData, qualifierId: m_valuePip.Qualifier, usage: usage); return(result); }
/// <summary> /// Adds a fake write file pip that produces to the given destination path. /// </summary> public WriteFile AddWriteFilePip(AbsolutePath destinationPath) { Contract.Requires(destinationPath != null); FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationPath).CreateNextWrittenVersion(); PipData contents = PipDataBuilder.CreatePipData(m_context.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "content"); var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(m_context)); writeFile.PipId = AllocateNextPipId(); m_pips.Add(writeFile.PipId, writeFile); m_pathProducers.Add(destinationArtifact, writeFile); return(writeFile); }
private void VerifyNested1(PipFragment fragment) { PipData nestedData = fragment.GetNestedFragmentValue(); XAssert.AreEqual(NumStandardBlockFragments + 1, nestedData.FragmentCount); using (var nestedEnumerator = nestedData.GetEnumerator()) { var localEnumerator = nestedEnumerator; VerifyStandardBlock(ref localEnumerator); AssertMoveNext(ref localEnumerator, out fragment, PipFragmentType.StringLiteral); XAssert.AreEqual(UniqueEntry1, m_pathTable.StringTable.GetString(fragment.GetStringIdValue())); XAssert.IsFalse(localEnumerator.MoveNext()); } }
private void VerifyNested0(PipFragment fragment) { PipData nestedData = fragment.GetNestedFragmentValue(); XAssert.AreEqual(NumStandardBlockFragments + 1, nestedData.FragmentCount); using (var nestedEnumerator = nestedData.GetEnumerator()) { var localEnumerator = nestedEnumerator; AssertMoveNext(ref localEnumerator, out fragment, PipFragmentType.AbsolutePath); XAssert.AreEqual(m_uniqueEntry0, fragment.GetPathValue()); VerifyStandardBlock(ref localEnumerator); XAssert.IsFalse(localEnumerator.MoveNext()); } }
/// <nodoc /> public bool TrySealDirectory( AbsolutePath directoryRoot, SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> contents, SortedReadOnlyArray <DirectoryArtifact, OrdinalDirectoryArtifactComparer> outputDirectorycontents, SealDirectoryKind kind, string[] tags, string description, string[] patterns, out DirectoryArtifact sealedDirectory, bool scrub = false) { Contract.Requires(directoryRoot.IsValid); Contract.Requires(contents.IsValid); Contract.Requires(outputDirectorycontents.IsValid); PipData usage = PipDataBuilder.CreatePipData(Context.StringTable, string.Empty, PipDataFragmentEscaping.NoEscaping, description != null ? new PipDataAtom[] { description } : new PipDataAtom[] { "'", directoryRoot, "' [", contents.Length.ToString(CultureInfo.InvariantCulture), " files - ", outputDirectorycontents.Length.ToString(CultureInfo.InvariantCulture), " output directories]" }); var pip = new SealDirectory( directoryRoot, contents, outputDirectorycontents, kind, CreatePipProvenance(usage), ToStringIds(tags), ToStringIds(patterns), scrub); if (PipGraph != null) { sealedDirectory = PipGraph.AddSealDirectory(pip, GetValuePipId()); if (!sealedDirectory.IsValid) { return(false); } } else { sealedDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(directoryRoot); } return(true); }
/// <nodoc/> public bool TryComposeSharedOpaqueDirectory( AbsolutePath directoryRoot, IReadOnlyList <DirectoryArtifact> contents, SealDirectoryContentFilter?contentFilter, [CanBeNull] string description, [CanBeNull] string[] tags, out DirectoryArtifact sharedOpaqueDirectory) { Contract.Requires(directoryRoot.IsValid); Contract.Requires(contents != null); if (PipGraph == null) { sharedOpaqueDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(directoryRoot); return(true); } PipData usage = PipDataBuilder.CreatePipData(Context.StringTable, string.Empty, PipDataFragmentEscaping.NoEscaping, description != null ? new PipDataAtom[] { description } : new PipDataAtom[] { "'", directoryRoot, "' [", contents.Count.ToString(CultureInfo.InvariantCulture), " shared opaque directories, filter: ", contentFilter.HasValue ? $"'{contentFilter.Value.Regex}' (kind: {Enum.GetName(typeof(SealDirectoryContentFilter.ContentFilterKind), contentFilter.Value.Kind)})" : "''", "]" }); sharedOpaqueDirectory = PipGraph.ReserveSharedOpaqueDirectory(directoryRoot); var pip = new CompositeSharedOpaqueSealDirectory( directoryRoot, contents, CreatePipProvenance(usage), ToStringIds(tags), contentFilter); // The seal directory is ready to be initialized, since the directory artifact has been reserved already pip.SetDirectoryArtifact(sharedOpaqueDirectory); sharedOpaqueDirectory = PipGraph.AddSealDirectory(pip, GetValuePipId()); if (!sharedOpaqueDirectory.IsValid) { return(false); } return(true); }
internal void ProcessNestedArgument(PipData nested, StringBuilder builder) { foreach (var arg in nested) { switch (arg.FragmentType) { case PipFragmentType.StringLiteral: builder.AppendLine(arg.GetStringIdValue().ToString(Context.StringTable)); break; case PipFragmentType.AbsolutePath: builder.AppendLine(arg.GetPathValue().ToString(Context.PathTable)); break; case PipFragmentType.NestedFragment: builder.AppendLine(); builder.AppendLine(); ProcessNestedArgument(arg.GetNestedFragmentValue(), builder); break; } } }
/// <summary> /// Gets the command line arguments for the process. /// </summary> public PipData GetArgumentsDataFromProcess(Process process) { PipData arguments = process.Arguments; if (process.ResponseFile.IsValid) { var responseFileData = process.ResponseFileData; PipDataBuilder pipDataBuilder = new PipDataBuilder(StringTable); // Add all the arguments from the command line excluding the response file (the last fragment) foreach (var fragment in process.Arguments.Take(process.Arguments.FragmentCount - 1).Concat(responseFileData)) { Contract.Assume(fragment.FragmentType != PipFragmentType.Invalid); pipDataBuilder.Add(fragment); } arguments = pipDataBuilder.ToPipData(arguments.FragmentSeparator, arguments.FragmentEscaping); } return(arguments); }
private static IEnumerable <string> ConvertArgumentsToStringArray(PipData pipData, PathTable pathTable) { var result = new List <string>(); foreach (PipFragment fragment in pipData) { Contract.Assume(fragment.FragmentType != PipFragmentType.Invalid); string s = string.Empty; switch (fragment.FragmentType) { case PipFragmentType.StringLiteral: s = pathTable.StringTable.GetString(fragment.GetStringIdValue()); break; case PipFragmentType.AbsolutePath: s = fragment.GetPathValue().ToString(pathTable); break; case PipFragmentType.NestedFragment: s = fragment.GetNestedFragmentValue().ToString(pathTable); break; default: Contract.Assert(false, "Unhandled fragment type"); break; } if (pipData.FragmentEscaping == PipDataFragmentEscaping.CRuntimeArgumentRules) { s = CommandLineEscaping.EscapeAsCommandLineWord(s); } s += pathTable.StringTable.GetString(pipData.FragmentSeparator); result.Add(s); } return(result); }
public void PipDataReadWrite() { var pipData = m_pipDataBuilder.ToPipData(EnclosingSeparator, EnclosingEscaping); byte[] bytes = new byte[10000]; int startIndex = 23; int index = startIndex; foreach (var entry in pipData.Entries) { var expectedIndex = index + PipDataEntry.BinarySize; entry.Write(bytes, ref index); // Ensure the correct number of bytes are written Assert.Equal(expectedIndex, index); } var endIndex = index; index = startIndex; List <PipDataEntry> readEntries = new List <PipDataEntry>(); while (index < endIndex) { var expectedIndex = index + PipDataEntry.BinarySize; readEntries.Add(PipDataEntry.Read(bytes, ref index)); // Ensure the correct number of bytes are read Assert.Equal(expectedIndex, index); } var readPipData = PipData.CreateInternal(pipData.HeaderEntry, PipDataEntryList.FromEntries(readEntries), StringId.Invalid); VerifyFullPipData(readPipData); }
private bool TryConfigureProcessBuilder(ProcessBuilder processBuilder, NinjaNode node, QualifierId qualifierId) { SeparateExecutableFromCommands(node.Command, out string executable, out string args); if (!TryFindExecutablePath(executable, out AbsolutePath exePath)) { Tracing.Logger.Log.InvalidExecutablePath(m_context.LoggingContext, Location.FromFile(m_specPath.ToString(m_context.PathTable)), node.Command); return(false); } FileArtifact prExeArtifact = FileArtifact.CreateSourceFile(exePath); processBuilder.Executable = prExeArtifact; processBuilder.AddInputFile(prExeArtifact); using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder()) { var pipDataBuilder = pipDataBuilderWrapper.Instance; pipDataBuilder.Add(args); processBuilder.ArgumentsBuilder.Add(pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping)); } if (node.ResponseFile.HasValue) { using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder()) { var pipDataBuilder = pipDataBuilderWrapper.Instance; pipDataBuilder.Add(node.ResponseFile?.Content); PipData responseFileData = pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping); // We tell the process builder to write the response file but to not add any arguments to the process (requiresExplicitArgument = false) // because that information is already in node.Command var rspFileSpec = ResponseFileSpecification.Builder() .ExplicitData(responseFileData) .RequiresArgument(false) .ExplicitPath((AbsolutePath)node.ResponseFile?.Path) .Build(); processBuilder.SetResponseFileSpecification(rspFileSpec); } } // TODO: Maybe a better description. Add ninja description or change command for input/outputs processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {node.Rule} - {executable} :: [{node.Command}]")); processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads | Process.Options.OutputsMustRemainWritable | Process.Options.OutputsMustRemainWritable; processBuilder.EnableTempDirectory(); // Working directory - the directory containing the ninja spec file // Ninja generators may express paths relative to this processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(m_specPath.GetParent(m_context.PathTable)); // Untrack directories UntrackFilesAndDirectories(processBuilder); // Allow some surviving child process AddRequiredSurvivingChildren(processBuilder); // Environment variables SetEnvironmentVariables(processBuilder, node); return(true); }
private static string CreateString(PipData value, PathTable pathTable) { return(value.IsValid ? value.ToString(pathTable) : null); }
public XElement CreateRow(string key, PipData value) { return(value.IsValid ? CreateRow(key, value.ToString(m_pathTable)) : null); }
/// <summary> /// Configure the processBuilder's ResponseFile and ResponseFileData according to this specification. /// This method mutates the processBuilder, changing its argumentsBuilder, ResponseFile and ResponseFileData accordingly. /// We return the arguments to be passed to the process. /// </summary> internal PipData SplitArgumentsAndCreateResponseFileIfNeeded(ProcessBuilder processBuilder, DirectoryArtifact defaultDirectory, PathTable pathTable) { PipData arguments = default; // We'll return the actual arguments to be passed to the process. var argumentsBuilder = processBuilder.ArgumentsBuilder; var cutoffArg = m_firstArg; // We will create a response file in the following cases: // 1. If an explicit response file content was specified, either by having m_explicitData or // by having m_forceCreation = true // 2. If the argument line is too long (longer than MaxCommandLineLength) and m_firstArg is not the default. // An additional argument is added to the process specifying the response file location, prefixed // by m_prefix, unless m_requiresArgument was set to false. if (!m_firstArg.IsDefault || m_explicitData.IsValid) { bool argumentLineTooLong = false; if (!m_forceCreation) { // make a response file only if the command-line is too long arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules); // Normalize choice to use response file by assuming paths are of length max path with a space. This will // ensure there are no cases where changing the root will change whether a response file is used. int cmdLineLength = arguments.GetMaxPossibleLength(pathTable.StringTable); argumentLineTooLong = cmdLineLength > ProcessBuilder.MaxCommandLineLength; } if (m_forceCreation || argumentLineTooLong || m_explicitData.IsValid) { // add the explicit contents if we have to if (m_explicitData.IsValid) { if (!argumentLineTooLong) { // If there was no overflow, we mark 'here' as the starting // point for the response file before adding the explicit data as an 'argument'. cutoffArg = argumentsBuilder.CreateCursor(); } argumentsBuilder.Add(m_explicitData); } // create a pip data for the stuff in the response file processBuilder.ResponseFileData = argumentsBuilder.ToPipData(Environment.NewLine, PipDataFragmentEscaping.CRuntimeArgumentRules, cutoffArg); // generate the file processBuilder.ResponseFile = FileArtifact.CreateSourceFile(GetResponseFilePath(defaultDirectory, pathTable)); argumentsBuilder.TrimEnd(cutoffArg); processBuilder.AddUntrackedFile(processBuilder.ResponseFile); if (m_requiresArgument) { if (string.IsNullOrEmpty(m_prefix)) { argumentsBuilder.Add(processBuilder.ResponseFile); } else { using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, pathTable.StringTable.Empty)) { argumentsBuilder.Add(m_prefix); argumentsBuilder.Add(processBuilder.ResponseFile); } } } arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules); } } else { arguments = argumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules); } return(arguments); }
private void IterateCscArguments(Project project, PipData arguments, bool isNested = false) { Action <object> action = null; foreach (var arg in arguments) { var type = arg.FragmentType; if (action != null) { action(GetObjectValue(arg)); action = null; } else if (type == PipFragmentType.AbsolutePath && !isNested) { var path = GetPathValue(arg); AddSourceItem(path, project, "Compile"); } else if (type == PipFragmentType.StringLiteral) { var strValue = arg.GetStringIdValue().ToString(Context.StringTable); switch (strValue) { case "/analyzer:": action = (obj) => project.AddItem("Analyzer", (AbsolutePath)obj); break; case "/r:": case "/link:": action = (obj) => project.RawReferences.Add((AbsolutePath)obj); break; case "/langversion:": action = (obj) => project.SetProperty("LangVersion", (string)obj); break; case "/target:": action = (obj) => project.SetProperty("OutputType", (string)obj); break; case "/keyfile:": action = (obj) => { project.SetProperty("AssemblyOriginatorKeyFile", (AbsolutePath)obj); project.SetProperty("SignAssembly", bool.TrueString); project.SetProperty("DelaySign", bool.FalseString); }; break; case "/define:": action = (obj) => project.SetProperty("DefineConstants", (string)obj); break; case "/unsafe": action = (obj) => { if ((string)obj == "+") { project.SetProperty("AllowUnsafeBlocks", bool.TrueString); } }; break; case "/unsafe+": project.SetProperty("AllowUnsafeBlocks", bool.TrueString); break; case "/recurse:": action = (obj) => { var expandedPathWithWildCard = ((PipData)obj).ToString(Context.PathTable); var msbuildMatchString = expandedPathWithWildCard.Replace("*", "**"); project.AddItem("Compile", msbuildMatchString); // if (!Context.CanWriteToSrc) // { // // We only need link if the file goes outside of the projects cone. // // If the item path and the link are the same path. VisualStudio will not render the file in the project tree. // RelativePath relativePath; // if (SpecDirectory.TryGetRelative(Context.PathTable, path, out relativePath) && relativePath.GetAtoms().Length > 1) // { // item.SetMetadata("Link", relativePath.ToString(Context.StringTable)); // } // } }; break; case "/out:": action = (obj) => { var assemblyPath = (AbsolutePath)obj; // TODO: There should be only one assembly name for a csproj file var outputFileWithoutExt = assemblyPath.GetName(Context.PathTable).RemoveExtension(Context.StringTable); project.SetProperty("AssemblyName", outputFileWithoutExt); project.SetProperty("RootNamespace", outputFileWithoutExt); var outputDir = assemblyPath.GetParent(Context.PathTable); SetOutputDirectory(project, outputDir, OutputDirectoryType.Build); }; break; case "/ruleset:": action = (obj) => { project.SetProperty("CodeAnalysisRuleSet", (AbsolutePath)obj); project.SetProperty("RunCodeAnalysis", bool.TrueString); }; break; case "/additionalfile:": action = (obj) => project.AddItem("AdditionalFiles", (AbsolutePath)obj); break; case "/features:": action = obj => { var features = ((PipData)obj).ToString(Context.PathTable); project.SetProperty("Features", features); }; break; case "/nullable": action = (obj) => { if ((string)obj == "+") { project.SetProperty("Nullable", "enable"); } }; break; default: if (strValue.StartsWith("/target:", StringComparison.OrdinalIgnoreCase)) { // BuildXL XML specific project.SetProperty("OutputType", strValue.Substring(8)); } break; } } else if (type == PipFragmentType.NestedFragment) { IterateCscArguments(project, arg.GetNestedFragmentValue(), true); } } }
public async Task Stress() { const int N = 5; const int M = N * N; var context = BuildXLContext.CreateInstanceForTesting(); var loggingContext = CreateLoggingContextForTest(); var pathTable = context.PathTable; using (var tempFiles = new TempFileStorage(canGetFileNames: true)) { var config = ConfigHelpers.CreateDefault(pathTable, tempFiles.GetUniqueFileName(), tempFiles); using (var pipTable = new PipTable( context.PathTable, context.SymbolTable, initialBufferSize: 1024, maxDegreeOfParallelism: (Environment.ProcessorCount + 2) / 3, debug: false)) { var executionEnvironment = new PipQueueTestExecutionEnvironment( context, config, pipTable, Path.Combine(TestOutputDirectory, "temp"), TryGetSubstSourceAndTarget(out string substSource, out string substTarget) ? (substSource, substTarget) : default((string, string)?), GetSandboxConnection()); Func <RunnablePip, Task <PipResult> > taskFactory = async(runnablePip) => { PipResult result; var operationTracker = new OperationTracker(runnablePip.LoggingContext); var pip = runnablePip.Pip; using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, runnablePip.LoggingContext)) { result = await TestPipExecutor.ExecuteAsync(operationContext, executionEnvironment, pip); } executionEnvironment.MarkExecuted(pip); return(result); }; string executable = CmdHelper.OsShellExe; FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable)); // This is the only file artifact we reference without a producer. Rather than scheduling a hashing pip, let's just invent one (so fingerprinting can succeed). executionEnvironment.AddWellKnownFile(executableArtifact, WellKnownContentHashes.UntrackedFile); using (var phase1PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule)) { // phase 1: create some files var baseFileArtifacts = new List <FileArtifact>(); for (int i = 0; i < N; i++) { string destination = tempFiles.GetUniqueFileName(); AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination); FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion(); baseFileArtifacts.Add(destinationArtifact); PipData contents = PipDataBuilder.CreatePipData( context.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, i.ToString(CultureInfo.InvariantCulture)); var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context)); var pipId = pipTable.Add((uint)(i + 1), writeFile); var contentHash = ContentHashingUtilities.HashString(contents.ToString(pathTable)); executionEnvironment.AddExpectedWrite(writeFile, destinationArtifact, contentHash); var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactory, 0); runnable.Start(new OperationTracker(loggingContext), loggingContext); runnable.SetDispatcherKind(DispatcherKind.IO); phase1PipQueue.Enqueue(runnable); } phase1PipQueue.SetAsFinalized(); phase1PipQueue.DrainQueues(); await Task.WhenAll( Enumerable.Range(0, 2).Select( async range => { using (var phase2PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule)) { // phase 2: do some more with those files var pips = new ConcurrentDictionary <PipId, Tuple <string, int> >(); var checkerTasks = new ConcurrentQueue <Task>(); Action <PipId, Task <PipResult> > callback = (id, task) => { XAssert.IsTrue(task.Status == TaskStatus.RanToCompletion); XAssert.IsFalse(task.Result.Status.IndicatesFailure()); Tuple <string, int> t; if (!pips.TryRemove(id, out t)) { XAssert.Fail(); } checkerTasks.Enqueue( Task.Run( () => { string actual = File.ReadAllText(t.Item1).Trim(); // TODO: Make this async XAssert.AreEqual(actual, t.Item2.ToString()); })); }; var r = new Random(0); for (int i = 0; i < M; i++) { int sourceIndex = r.Next(baseFileArtifacts.Count); FileArtifact sourceArtifact = baseFileArtifacts[sourceIndex]; string destination = tempFiles.GetUniqueFileName(); AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination); FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion(); Pip pip; DispatcherKind queueKind; switch (r.Next(2)) { case 0: pip = new CopyFile(sourceArtifact, destinationArtifact, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context)); queueKind = DispatcherKind.IO; executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact)); break; case 1: string workingDirectory = OperatingSystemHelper.IsUnixOS ? "/tmp" : Environment.GetFolderPath(Environment.SpecialFolder.Windows); AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory); var pipData = OperatingSystemHelper.IsUnixOS ? PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "-c", "'", "cp", sourceArtifact, destinationArtifact, "'") : PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "/d", "/c", "copy", "/B", sourceArtifact, destinationArtifact); queueKind = DispatcherKind.CPU; pip = new Process( executableArtifact, workingDirectoryAbsolutePath, pipData, FileArtifact.Invalid, PipData.Invalid, ReadOnlyArray <EnvironmentVariable> .Empty, FileArtifact.Invalid, FileArtifact.Invalid, FileArtifact.Invalid, tempFiles.GetUniqueDirectory(pathTable), null, null, ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact, sourceArtifact), ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(destinationArtifact.WithAttributes()), ReadOnlyArray <DirectoryArtifact> .Empty, ReadOnlyArray <DirectoryArtifact> .Empty, ReadOnlyArray <PipId> .Empty, ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencies(pathTable)), ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(pathTable)), ReadOnlyArray <StringId> .Empty, ReadOnlyArray <int> .Empty, ReadOnlyArray <ProcessSemaphoreInfo> .Empty, provenance: PipProvenance.CreateDummy(context), toolDescription: StringId.Invalid, additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty); executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact)); break; default: Contract.Assert(false); continue; } var pipId = pipTable.Add((uint)((range *M) + N + i + 1), pip); Func <RunnablePip, Task> taskFactoryWithCallback = async(runnablePip) => { var task = taskFactory(runnablePip); var pipResult = await task; callback(pipId, task); }; var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactoryWithCallback, 0); runnable.Start(new OperationTracker(loggingContext), loggingContext); runnable.SetDispatcherKind(queueKind); phase2PipQueue.Enqueue(runnable); if (!pips.TryAdd(pipId, Tuple.Create(destination, sourceIndex))) { Contract.Assert(false); } } phase2PipQueue.SetAsFinalized(); phase2PipQueue.DrainQueues(); XAssert.AreEqual(0, pips.Count); await Task.WhenAll(checkerTasks); } })); } } } }