public void ComplexTestCombinedWithSerializer() { var topic = TestExecuter.GetTopic(); var builder = new ProcessBuilder() { InputProcess = TestData.Person(topic), Mutators = new MutatorList() { new InPlaceConvertMutator(topic, null) { Columns = new[] { "birthDate" }, TypeConverter = new DateConverterAuto(new CultureInfo("hu-HU")), ActionIfInvalid = InvalidValueAction.Throw, }, new ExplodeMutator(topic, null) { RowCreator = (proc, row) => { var newRow = new SlimRow { ["personModel"] = new PersonModel() { Id = row.GetAs <int>("id"), Name = row.GetAs <string>("name"), Age = row.GetAs <int?>("age"), BirthDate = row.GetAs <DateTime?>("birthDate"), } }; return(new[] { newRow }); }, }, new DataContractXmlSerializerMutator <PersonModel>(topic, "serialize to XML byte[]") { ColumnConfiguration = new ColumnCopyConfiguration("personModel", "personModelXml"), }, new RemoveColumnMutator(topic, null) { Columns = new[] { "personModel" }, }, new InPlaceConvertMutator(topic, "deserialize from XML byte[]") { Columns = new[] { "personModelXml" }, TypeConverter = new DataContractXmlDeSerializerConverter <PersonModel>(), }, new RenameColumnMutator(topic, null) { ColumnConfiguration = new List <ColumnRenameConfiguration>() { new ColumnRenameConfiguration("personModelXml", "personModel"), }, }, new ExplodeMutator(topic, null) { RowCreator = (proc, row) => { var personModel = row.GetAs <PersonModel>("personModel"); var newRow = new SlimRow() { ["id"] = personModel.Id, ["name"] = personModel.Name, ["age"] = personModel.Age, ["birthDate"] = personModel.BirthDate, }; return(new[] { newRow }); }, }, }, }; var result = TestExecuter.Execute(builder); Assert.AreEqual(7, result.MutatedRows.Count); Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >() { new CaseInsensitiveStringKeyDictionary <object>() { ["id"] = 0, ["name"] = "A", ["age"] = 17, ["birthDate"] = new DateTime(2010, 12, 9, 0, 0, 0, 0) }, new CaseInsensitiveStringKeyDictionary <object>() { ["id"] = 1, ["name"] = "B", ["age"] = 8, ["birthDate"] = new DateTime(2011, 2, 1, 0, 0, 0, 0) }, new CaseInsensitiveStringKeyDictionary <object>() { ["id"] = 2, ["name"] = "C", ["age"] = 27, ["birthDate"] = new DateTime(2014, 1, 21, 0, 0, 0, 0) }, new CaseInsensitiveStringKeyDictionary <object>() { ["id"] = 3, ["name"] = "D", ["age"] = 39, ["birthDate"] = new DateTime(2018, 7, 11, 0, 0, 0, 0) }, new CaseInsensitiveStringKeyDictionary <object>() { ["id"] = 4, ["name"] = "E", ["age"] = -3 }, new CaseInsensitiveStringKeyDictionary <object>() { ["id"] = 5, ["name"] = "A", ["age"] = 11, ["birthDate"] = new DateTime(2013, 5, 15, 0, 0, 0, 0) }, new CaseInsensitiveStringKeyDictionary <object>() { ["id"] = 6, ["name"] = "fake", ["birthDate"] = new DateTime(2018, 1, 9, 0, 0, 0, 0) } }); var exceptions = topic.Context.GetExceptions(); Assert.AreEqual(0, exceptions.Count); }
private bool TryConfigureProcessBuilder( ProcessBuilder processBuilder, PipConstructionHelper pipConstructionHelper, ProjectWithPredictions project, QualifierId qualifierId, out AbsolutePath outputResultCacheFile, out string failureDetail) { outputResultCacheFile = AbsolutePath.Invalid; if (!TrySetBuildToolExecutor(pipConstructionHelper, processBuilder, project)) { failureDetail = "Failed to construct tooldefinition"; return(false); } // Working directory - the directory where the project file lives. processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.FullPath.GetParent(PathTable)); // We allow undeclared inputs to be read processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads; // Run in a container if specified if (m_resolverSettings.RunInContainer) { processBuilder.Options |= Process.Options.NeedsToRunInContainer; processBuilder.ContainerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs; } // By default the double write policy is to allow same content double writes. processBuilder.DoubleWritePolicy |= m_resolverSettings.DoubleWritePolicy ?? DoubleWritePolicy.AllowSameContentDoubleWrites; SetUntrackedFilesAndDirectories(processBuilder); // Add the log directory and its corresponding files var qualifier = m_context.QualifierTable.GetQualifier(qualifierId); AbsolutePath logDirectory = GetLogDirectory(project, qualifier); processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.log"), FileExistence.Optional); processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.wrn"), FileExistence.Optional); processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.err"), FileExistence.Optional); processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.prf"), FileExistence.Optional); if (m_resolverSettings.EnableBinLogTracing == true) { processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.binlog"), FileExistence.Optional); } // Unless the legacy non-isolated mode is explicitly specified, the project builds in isolation, and therefore // it produces an output cache file. This file is placed on the (unique) object directory for this project if (m_resolverSettings.UseLegacyProjectIsolation != true) { var objectDirectory = pipConstructionHelper.GetUniqueObjectDirectory(project.FullPath.GetName(PathTable)); outputResultCacheFile = objectDirectory.Path.Combine(PathTable, PathAtom.Create(PathTable.StringTable, OutputCacheFileName)); processBuilder.AddOutputFile(outputResultCacheFile, FileExistence.Required); } // Path to the project processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(project.FullPath)); // Response file with the rest of the arguments var rspFileSpec = ResponseFileSpecification.Builder() .AllowForRemainingArguments(processBuilder.ArgumentsBuilder.CreateCursor()) .ForceCreation(true) .Prefix("@") .Build(); processBuilder.SetResponseFileSpecification(rspFileSpec); if (!TryAddMsBuildArguments(project, processBuilder.ArgumentsBuilder, logDirectory, outputResultCacheFile, out failureDetail)) { return(false); } // Q_SESSION_GUID is used to provide a unique build GUID to build tools and scripts. // It'll cause full cache misses if we try to hash it as an input, however, so exclude. processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, BuildEnvironmentConstants.QSessionGuidEnvVar)); // mspdbsrv: _MSPDBSRV_ENDPOINT_ sets up one mspdbsrv.exe instance per build target execution. // However this process will live beyond the build.cmd or msbuild.exe call. // Allow the pip job object to clean the process without complaint. // // vctip.exe: On any compile error this telemetry upload exe will be run as a detached process. // Just let it be killed. // TODO: Can we stop it running? https://stackoverflow.microsoft.com/questions/74425/how-to-disable-vctip-exe-in-vc14 // // conhost.exe: This process needs a little bit more time to finish after the main process. We shouldn't be allowing // this one to survive, we just need the timeout to be slightly more than zero. This will also be beneficial to other // arbitrary processeses that need a little bit more time. But, apparently, setting a timeout has a perf impact that is // being investigated. TODO: revisit this once this is fixed. // // All child processes: Don't wait to kill the processes. // CODESYNC: CloudBuild repo TrackerExecutor.cs "info.NestedProcessTerminationTimeout = TimeSpan.Zero" processBuilder.AllowedSurvivingChildProcessNames = ReadOnlyArray <PathAtom> .FromWithoutCopy( PathAtom.Create(m_context.StringTable, "mspdbsrv.exe"), PathAtom.Create(m_context.StringTable, "vctip.exe"), PathAtom.Create(m_context.StringTable, "conhost.exe"), PathAtom.Create(m_context.StringTable, "VBCSCompiler.exe")); processBuilder.NestedProcessTerminationTimeout = TimeSpan.Zero; SetProcessEnvironmentVariables(CreateEnvironment(logDirectory, project), processBuilder); failureDetail = string.Empty; return(true); }
/// <summary> /// Gets IPC process builder. /// </summary> /// <returns></returns> public ProcessBuilder GetIpcProcessBuilder() => ProcessBuilder.CreateForTesting(Context.PathTable);
private bool TryScheduleIpcPip(Context context, ObjectLiteral obj, bool allowUndefinedTargetService, bool isServiceFinalization, out FileArtifact outputFile, out PipId pipId) { // IpcClientInfo IIpcMoniker moniker = Converter.ExtractRef <IIpcMoniker>(obj, m_ipcSendMoniker, allowUndefined: false); int? numRetries = Converter.ExtractNumber(obj, m_ipcSendMaxConnectRetries, allowUndefined: true); int? retryDelayMillis = Converter.ExtractNumber(obj, m_ipcSendConnectRetryDelayMillis, allowUndefined: true); var clientConfig = new ClientConfig(numRetries, retryDelayMillis); var ipcClientInfo = new IpcClientInfo(moniker.ToStringId(context.StringTable), clientConfig); // target service pip PipId?servicePipId = Converter.ExtractValue <PipId>(obj, m_ipcSendTargetServicePip, allowUndefined: allowUndefinedTargetService); // arguments PipData arguments; ReadOnlyArray <FileArtifact> fileDependencies; ReadOnlyArray <DirectoryArtifact> directoryDependencies; using (var ipcProcessBuilder = ProcessBuilder.Create(context.PathTable, context.FrontEndContext.GetPipDataBuilder())) { // process arguments ArrayLiteral argumentsArrayLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendMessageBody); TransformerExecuteArgumentsProcessor.ProcessArguments(context, ipcProcessBuilder, argumentsArrayLiteral); // input file dependencies var dependenciesArray = Converter.ExtractArrayLiteral(obj, m_ipcSendDependencies, allowUndefined: true); if (dependenciesArray != null) { for (int i = 0; i < dependenciesArray.Length; i++) { ProcessImplicitDependency(ipcProcessBuilder, dependenciesArray[i], convContext: new ConversionContext(pos: i, objectCtx: dependenciesArray)); } } arguments = ipcProcessBuilder.ArgumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules); fileDependencies = ipcProcessBuilder.GetInputFilesSoFar(); directoryDependencies = ipcProcessBuilder.GetInputDirectoriesSoFar(); } // output AbsolutePath output = Converter.ExtractPath(obj, m_ipcSendOutputFile, allowUndefined: true); if (!output.IsValid) { output = context.GetPipConstructionHelper().GetUniqueObjectDirectory(m_ipcObjectFolderName).Path.Combine(context.PathTable, m_ipcOutputFileName); } // tags string[] tags = null; var tagsArray = Converter.ExtractArrayLiteral(obj, m_executeTags, allowUndefined: true); if (tagsArray != null && tagsArray.Count > 0) { tags = new string[tagsArray.Count]; for (int i = 0; i < tagsArray.Count; i++) { tags[i] = Converter.ExpectString(tagsArray[i], context: new ConversionContext(pos: i, objectCtx: tagsArray)); } } // skip materialization for files FileOrDirectoryArtifact[] skipMaterializationArtifacts = CollectionUtilities.EmptyArray <FileOrDirectoryArtifact>(); ArrayLiteral skipMaterializationLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendLazilyMaterializedDependencies, allowUndefined: true); if (skipMaterializationLiteral != null) { skipMaterializationArtifacts = new FileOrDirectoryArtifact[skipMaterializationLiteral.Length]; for (int i = 0; i < skipMaterializationLiteral.Length; i++) { Converter.ExpectFileOrStaticDirectory( skipMaterializationLiteral[i], out var fileArtifact, out var staticDirectory, context: new ConversionContext(pos: i, objectCtx: skipMaterializationLiteral)); Contract.Assert(fileArtifact.IsValid ^ staticDirectory != null); skipMaterializationArtifacts[i] = fileArtifact.IsValid ? FileOrDirectoryArtifact.Create(fileArtifact) : FileOrDirectoryArtifact.Create(staticDirectory.Root); } } // must run on master var mustRunOnMaster = Converter.ExtractOptionalBoolean(obj, m_ipcSendMustRunOnMaster) == true; outputFile = FileArtifact.CreateOutputFile(output); // create IPC pip and add it to the graph return(context.GetPipConstructionHelper().TryAddIpc( ipcClientInfo, arguments, outputFile, servicePipDependencies: servicePipId != null ? ReadOnlyArray <PipId> .From(new[] { servicePipId.Value }) : ReadOnlyArray <PipId> .Empty, fileDependencies: fileDependencies, directoryDependencies: directoryDependencies, skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .FromWithoutCopy(skipMaterializationArtifacts), isServiceFinalization: isServiceFinalization, mustRunOnMaster: mustRunOnMaster, tags: tags, out pipId)); }
/// <summary> /// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project /// </summary> /// <remarks> /// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs /// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that /// otherwise will likely be just a shared opaque output at the root). /// </remarks> private void ProcessInputs( ProjectWithPredictions project, ProcessBuilder processBuilder) { // Predicted output directories for all direct dependencies, plus the output directories for the given project itself var knownOutputDirectories = project.ProjectReferences.SelectMany(reference => reference.PredictedOutputFolders).Union(project.PredictedOutputFolders); // Add all predicted inputs that are recognized as true source files // This is done to make the weak fingerprint stronger. Pips are scheduled so undeclared source reads are allowed. This means // we don't actually need accurate (or in fact any) input predictions to run successfully. But we are trying to avoid the degenerate case // of a very small weak fingerprint with too many candidates, that can slow down two-phase cache look-up. foreach (AbsolutePath buildInput in project.PredictedInputFiles) { // If any of the predicted inputs is under the predicted output folder of a dependency, then there is a very good chance the predicted input is actually an intermediate file // In that case, don't add the input as a source file to stay on the safe side. Otherwise we will have a file that is both declared as a source file and contained in a directory // dependency. if (knownOutputDirectories.Any(outputFolder => buildInput.IsWithin(PathTable, outputFolder))) { continue; } // If any of the predicted inputs is under an untracked directory scope, don't add it as an input if (processBuilder.GetUntrackedDirectoryScopesSoFar().Any(untrackedDirectory => buildInput.IsWithin(PathTable, untrackedDirectory))) { continue; } processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput)); } IEnumerable <ProjectWithPredictions> references; // The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in if (m_resolverSettings.EnableTransitiveProjectReferences == true) { // In this case all the transitive closure is automatically exposed to the project as direct references var transitiveReferences = new HashSet <ProjectWithPredictions>(); ComputeTransitiveDependenciesFor(project, transitiveReferences); references = transitiveReferences; } else { // Only direct dependencies are declared. // Add all known explicit inputs from project references. But rule out // projects that have a known empty list of targets: those projects are not scheduled, so // there is nothing to consume from them. references = project.ProjectReferences.Where(projectReference => !projectReference.PredictedTargetsToExecute.TargetsAreKnownToBeEmpty); } var argumentsBuilder = processBuilder.ArgumentsBuilder; foreach (ProjectWithPredictions projectReference in references) { bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out MSBuildProjectOutputs projectOutputs); if (!outputsPresent) { Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}"); } // Add all known output directories foreach (StaticDirectory output in projectOutputs.OutputDirectories) { processBuilder.AddInputDirectory(output.Root); } // If the dependency was built in isolation, this project needs to access the generated cache files if (projectOutputs.BuildsInIsolation) { var outputCache = projectOutputs.OutputCacheFile; processBuilder.AddInputFile(outputCache); // Instruct MSBuild to use the cache file from the associated dependency as an input. // Flag /irc is the short form of /inputResultsCaches, and part of MSBuild 'build in isolation' mode. using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty)) { argumentsBuilder.Add(PipDataAtom.FromString("/irc:")); argumentsBuilder.Add(PipDataAtom.FromAbsolutePath(outputCache)); } } } }
private void ButtonCreateMetadata_Click(object sender, EventArgs e) { ProcessBuilder.EditTransition(GetTransition()); MessageBox.Show("Process metadata created successfully."); }
private bool TryExecuteArgumentsToPipBuilder( ProjectWithPredictions project, QualifierId qualifierId, out string failureDetail, out Process scheduledProcess) { // We create a pip construction helper for each project var pipConstructionHelper = GetPipConstructionHelperForProject(project, qualifierId); using (var processBuilder = ProcessBuilder.Create(PathTable, m_context.GetPipDataBuilder())) { // Configure the process to add an assortment of settings: arguments, response file, etc. if (!TryConfigureProcessBuilder(processBuilder, pipConstructionHelper, project, qualifierId, out AbsolutePath outputResultCacheFile, out failureDetail)) { scheduledProcess = null; return(false); } // Process all predicted outputs and inputs, including the predicted project dependencies ProcessOutputs(project, processBuilder); ProcessInputs(project, processBuilder); // Try to schedule the process pip if (!pipConstructionHelper.TryAddProcess(processBuilder, out ProcessOutputs outputs, out scheduledProcess)) { failureDetail = "Failed to schedule the pip"; return(false); } // Add the computed outputs for this project, so dependencies can consume it var outputDirectories = outputs.GetOutputDirectories(); // A valid output cache path indicates that the project is building in isolation MSBuildProjectOutputs projectOutputs; if (outputResultCacheFile == AbsolutePath.Invalid) { projectOutputs = MSBuildProjectOutputs.CreateLegacy(outputDirectories); } else { var success = outputs.TryGetOutputFile(outputResultCacheFile, out FileArtifact cacheFileArtifact); if (!success) { Contract.Assert(false, I($"The output cache file {outputResultCacheFile.ToString(PathTable)} should be part of the project {project.FullPath.ToString(PathTable)} outputs.")); } projectOutputs = MSBuildProjectOutputs.CreateIsolated(outputDirectories, cacheFileArtifact); } // If the project is not implementing the target protocol, emit corresponding warn/verbose if (!project.ImplementsTargetProtocol) { if (project.ProjectReferences.Count != 0) { // Let's warn about this. Targets of the referenced projects may not be accurate Tracing.Logger.Log.ProjectIsNotSpecifyingTheProjectReferenceProtocol( m_context.LoggingContext, Location.FromFile(project.FullPath.ToString(PathTable)), project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable)); } else { // Just a verbose message in this case Tracing.Logger.Log.LeafProjectIsNotSpecifyingTheProjectReferenceProtocol( m_context.LoggingContext, Location.FromFile(project.FullPath.ToString(PathTable)), project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable)); } } // Warn if default targets were appended to the targets to execute if (project.PredictedTargetsToExecute.IsDefaultTargetsAppended) { Tracing.Logger.Log.ProjectPredictedTargetsAlsoContainDefaultTargets( m_context.LoggingContext, Location.FromFile(project.FullPath.ToString(PathTable)), project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable), $"[{string.Join(";", project.PredictedTargetsToExecute.AppendedDefaultTargets)}]"); } m_processOutputsPerProject[project] = projectOutputs; failureDetail = string.Empty; return(true); } }
private void CreateProcessFromControls() { ProcessBuilder.BuildProcess(Guid.NewGuid().ToString(), TextEditProcessName.Text, TextEditProcessCaption.Text, 2 /* WithoutState */); }
private void ConfigureProcessBuilder( ProcessBuilder processBuilder, RushProject project) { SetCmdTool(processBuilder, project); // Working directory - the directory where the project file lives. processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder); // We allow undeclared inputs to be read processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads; // We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient // to avoid a large number of path sets processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation; // Try to preserve path set casing since many JavaScript projects deal with paths in a case-sensitive way // Otherwise in Windows we force path sets to be all uppercase processBuilder.Options |= Process.Options.PreservePathSetCasing; // By default the double write policy is to allow same content double writes. processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites; // Untrack the user profile. The corresponding mount is already configured for not tracking source files, and with allowed undeclared source reads, // any attempt to read into the user profile will fail to compute its corresponding hash processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile))); // If dependencies are tracked with the shrinkwrap-deps file, then untrack everything under the Rush common temp folder, where all package // dependencies are placed if (m_resolverSettings.TrackDependenciesWithShrinkwrapDepsFile == true) { processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(m_rushConfiguration.CommonTempFolder)); } // Add the associated build script name as a tag, so filtering on 'build' or 'test' can happen processBuilder.Tags = ReadOnlyArray <StringId> .FromWithoutCopy(new[] { StringId.Create(m_context.StringTable, project.ScriptCommandName) }); PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings); var logDirectory = GetLogDirectory(project); processBuilder.SetStandardOutputFile(logDirectory.Combine(m_context.PathTable, "build.log")); processBuilder.SetStandardErrorFile(logDirectory.Combine(m_context.PathTable, "error.log")); using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " ")) { processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C")); using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, " ")) { // Execute the command and redirect the output to a designated log file processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.ScriptCommand)); // If we need to append arguments to the script command, do it here if (m_customCommands.TryGetValue(project.ScriptCommandName, out IReadOnlyList <RushArgument> extraArguments)) { foreach (RushArgument value in extraArguments) { AddRushArgumentToBuilder(processBuilder.ArgumentsBuilder, value); } } } } FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable); }
public override void execute(ActivityExecution execution) { readFields(execution); IList <string> argList = new List <string>(); argList.Add(commandStr); if (!string.ReferenceEquals(arg1Str, null)) { argList.Add(arg1Str); } if (!string.ReferenceEquals(arg2Str, null)) { argList.Add(arg2Str); } if (!string.ReferenceEquals(arg3Str, null)) { argList.Add(arg3Str); } if (!string.ReferenceEquals(arg4Str, null)) { argList.Add(arg4Str); } if (!string.ReferenceEquals(arg5Str, null)) { argList.Add(arg5Str); } ProcessBuilder processBuilder = new ProcessBuilder(argList); try { processBuilder.redirectErrorStream(redirectErrorFlag); if (cleanEnvBoolan.Value) { IDictionary <string, string> env = processBuilder.environment(); env.Clear(); } if (!string.ReferenceEquals(directoryStr, null) && directoryStr.Length > 0) { processBuilder.directory(new File(directoryStr)); } Process process = processBuilder.start(); if (waitFlag.Value) { int errorCode = process.waitFor(); if (!string.ReferenceEquals(resultVariableStr, null)) { string result = convertStreamToStr(process.InputStream); execution.setVariable(resultVariableStr, result); } if (!string.ReferenceEquals(errorCodeVariableStr, null)) { execution.setVariable(errorCodeVariableStr, Convert.ToString(errorCode)); } } } catch (Exception e) { throw LOG.shellExecutionException(e); } leave(execution); }
/// <summary> /// fork ourselves in a new jvm. sets -Dtests.crashmode=true </summary> public virtual void ForkTest() { IList<string> cmd = new List<string>(); cmd.Add(System.getProperty("java.home") + System.getProperty("file.separator") + "bin" + System.getProperty("file.separator") + "java"); cmd.Add("-Xmx512m"); cmd.Add("-Dtests.crashmode=true"); // passing NIGHTLY to this test makes it run for much longer, easier to catch it in the act... cmd.Add("-Dtests.nightly=true"); cmd.Add("-DtempDir=" + TempDir.Path); cmd.Add("-Dtests.seed=" + SeedUtils.formatSeed(Random().NextLong())); cmd.Add("-ea"); cmd.Add("-cp"); cmd.Add(System.getProperty("java.class.path")); cmd.Add("org.junit.runner.JUnitCore"); cmd.Add(this.GetType().Name); ProcessBuilder pb = new ProcessBuilder(cmd); pb.directory(TempDir); pb.redirectErrorStream(true); Process p = pb.Start(); // We pump everything to stderr. PrintStream childOut = System.err; Thread stdoutPumper = ThreadPumper.Start(p.InputStream, childOut); Thread stderrPumper = ThreadPumper.Start(p.ErrorStream, childOut); if (VERBOSE) { childOut.println(">>> Begin subprocess output"); } p.waitFor(); stdoutPumper.Join(); stderrPumper.Join(); if (VERBOSE) { childOut.println("<<< End subprocess output"); } }
public SearchTypeBuilder(ProcessBuilder processBuilder, TflSearchType searchType) { _processBuilder = processBuilder; _searchType = searchType; }
/// <summary> /// Forces every step of the process to use test options /// </summary> /// <param name="builder"></param> /// <returns></returns> public static ProcessBuilder ForceUseTestOptions(this ProcessBuilder builder) { builder.StepOptions = StepOptions.Test; return(builder); }
public static Process start(this ProcessBuilder processBuilder) { return(processBuilder.Start()); }
public MapBuilder(ProcessBuilder processBuilder, TflMap map) { _processBuilder = processBuilder; _map = map; }
public Process startInterpreter(String pythonExe) { return null; #if NYI Process p; try { ProcessBuilder builder = new ProcessBuilder(pythonExe, "-i", jsonizer); builder.redirectErrorStream(true); builder.redirectError(parserLog)); builder.redirectOutput(parserLog)); builder.environment().remove("PYTHONPATH"); p = builder.start(); } catch (Exception e) { _.msg("Failed to start: " + pythonExe); return null; } return p; #endif }
public RelationshipBuilder(ProcessBuilder processBuilder, TflRelationship relationship) { _processBuilder = processBuilder; _relationship = relationship; }
public void Test_DynamicActivities() { var process = new ProcessBuilder() .CreateActivity("act1") .CreateExecution("act1", _ => { return((c, t) => { // 获取需要动态分拆的数量 int number = t.Get("itemcount", 1); Transition nextTranistion = t.Destination.OutgoingTransitions?[0]; if (nextTranistion != null) { Transition dynamicTransition = null; // 第一个item使用原有的transition // 从第二个开始,添加transition并把 for (int i = 2; i <= number; i++) { dynamicTransition = new Transition { Source = nextTranistion.Source, Destination = nextTranistion.Destination }; nextTranistion.Source.AddOutgoingTransition(dynamicTransition); nextTranistion.Destination.AddIncomingTransition(dynamicTransition); } } return Task.CompletedTask; }); }) .CreateActivity("act2") .CreateExecution("act2", _ => { return((c, t) => { Console.WriteLine(c.Get("accepted", 0)); c.Set("accepted", c.Get("accepted", 0) + 1); return Task.CompletedTask; }); }) .CreateTransition("act1", "act2") .CreateActivity("act3") .CreateTransition("act2", "act3") .CreateActivity("act4") .CreateTransition("act3", "act4") .CreateActivity("act5") .CreateTransition("act4", "act5") .CreateExecution("act5", _ => { return((c, t) => { // TransitionState[] availableStates = new TransitionState[] { // TransitionState.Passed, TransitionState.Blocked // }; // if (t.Destination.IncomingTransitions.Any(ts => availableStates.Contains(ts.State) == false)) // { // t.CurrentTransition.SetState(TransitionState.Blocked); // } int number = c.Get("itemcount", 1); int accepted = c.Get("accepted", 0); if (number > accepted) { t.CurrentTransition.SetState(TransitionState.Blocked); } return Task.CompletedTask; }); }) .SetStart("act1") .Build(); var initData = new Dictionary <string, object>(); int itemcount = 5; initData["itemcount"] = itemcount; process.Start(initData); Assert.Equal(itemcount, process.Context.Get("accepted", 0)); }
private void LaunchGame(bool online) { Log.Info("Launching game..."); if (this.CheckCurrentProfile()) { if (online && this.Profiles.CurrentProfile != 0) { Messages.Show(this.Window, "CantPlayOnline", "Impossible", MessageBoxButton.OK, MessageBoxImage.Error); this.UiManager.Working = false; this.UiManager.ButtonsEnabled = true; this.UiManager.CanPlayOnline = this.UiManager.SelectedProfile == 0; } else { ProcessBuilder builder = new ProcessBuilder(); builder.WorkingDirectory = this.GtaPath; if (this.Settings.UseRph && File.Exists(Path.Combine(this.GtaPath, "RAGEPluginHook.exe"))) { Log.Info("Starting RAGE Plugin Hook process..."); builder.FilePath = Path.Combine(this.GtaPath, "RAGEPluginHook.exe"); } else if (this.IsSteamVersion()) { Log.Info("Starting steam game process..."); builder.FilePath = Path.Combine(this.SteamPath, "steam.exe"); builder.AddArgument("-applaunch 271590"); } else { Log.Info("Starting game process..."); builder.FilePath = Path.Combine(this.GtaPath, "GTAVLauncher.exe"); } Log.Info("Setting game language to " + this.Settings.GtaLanguage); builder.AddArgument("-uilanguage " + this.Settings.GtaLanguage); if (online) { builder.AddArgument("-StraightIntoFreemode"); } else if (this.Profiles.CurrentProfile != 0 && this.Settings.OfflineMode) { builder.AddArgument("-scOfflineOnly"); } Log.Info("Executing " + builder); builder.StartProcess(); this.CloseLauncher(); } } else { Log.Info("Aborting game launch."); this.UiManager.Working = false; this.UiManager.ButtonsEnabled = true; this.UiManager.CanPlayOnline = this.UiManager.SelectedProfile == 0; } }
/// <summary> /// Adds a step to the process that executes a request with a lazy loaded request data /// </summary> /// <param name="url"></param> /// <param name="method"></param> /// <param name="lazyData"></param> /// <returns></returns> public static ProcessBuilder AddStep(this ProcessBuilder builder, string url, HttpMethod method, Func <object> lazyData, StepOptions options = null) { var reference = Guid.Empty; return(builder.AddStep(url, method, lazyData, out reference, options)); }
public static DynamicStep GetDynamicStep(this ProcessBuilder builder, Guid reference) { return(builder.Steps.OfType <DynamicStep>().FirstOrDefault(s => s.Reference == reference)); }
/// <exception cref="System.IO.IOException"/> public virtual void Annotate(ICoreMap document) { // write input file in GUTime format IElement inputXML = ToInputXML(document); File inputFile = File.CreateTempFile("gutime", ".input"); //Document doc = new Document(inputXML); PrintWriter inputWriter = new PrintWriter(inputFile); inputWriter.Println(XMLUtils.NodeToString(inputXML, false)); // new XMLOutputter().output(inputXML, inputWriter); inputWriter.Close(); bool useFirstDate = (!document.ContainsKey(typeof(CoreAnnotations.CalendarAnnotation)) && !document.ContainsKey(typeof(CoreAnnotations.DocDateAnnotation))); List <string> args = new List <string>(); args.Add("perl"); args.Add("-I" + this.gutimePath.GetPath()); args.Add(new File(this.gutimePath, "TimeTag.pl").GetPath()); if (useFirstDate) { args.Add("-FDNW"); } args.Add(inputFile.GetPath()); // run GUTime on the input file ProcessBuilder process = new ProcessBuilder(args); StringWriter outputWriter = new StringWriter(); SystemUtils.Run(process, outputWriter, null); string output = outputWriter.GetBuffer().ToString(); Pattern docClose = Pattern.Compile("</DOC>.*", Pattern.Dotall); output = docClose.Matcher(output).ReplaceAll("</DOC>"); //The TimeTag.pl result file contains next tags which must be removed output = output.ReplaceAll("<lex.*?>", string.Empty); output = output.Replace("</lex>", string.Empty); output = output.Replace("<NG>", string.Empty); output = output.Replace("</NG>", string.Empty); output = output.Replace("<VG>", string.Empty); output = output.Replace("</VG>", string.Empty); output = output.Replace("<s>", string.Empty); output = output.Replace("</s>", string.Empty); // parse the GUTime output IElement outputXML; try { outputXML = XMLUtils.ParseElement(output); } catch (Exception ex) { throw new Exception(string.Format("error:\n%s\ninput:\n%s\noutput:\n%s", ex, IOUtils.SlurpFile(inputFile), output), ex); } /* * try { * outputXML = new SAXBuilder().build(new StringReader(output)).getRootElement(); * } catch (JDOMException e) { * throw new RuntimeException(String.format("error:\n%s\ninput:\n%s\noutput:\n%s", * e, IOUtils.slurpFile(inputFile), output)); * } */ inputFile.Delete(); // get Timex annotations IList <ICoreMap> timexAnns = ToTimexCoreMaps(outputXML, document); document.Set(typeof(TimeAnnotations.TimexAnnotations), timexAnns); if (outputResults) { System.Console.Out.WriteLine(timexAnns); } // align Timex annotations to sentences int timexIndex = 0; foreach (ICoreMap sentence in document.Get(typeof(CoreAnnotations.SentencesAnnotation))) { int sentBegin = BeginOffset(sentence); int sentEnd = EndOffset(sentence); // skip times before the sentence while (timexIndex < timexAnns.Count && BeginOffset(timexAnns[timexIndex]) < sentBegin) { ++timexIndex; } // determine times within the sentence int sublistBegin = timexIndex; int sublistEnd = timexIndex; while (timexIndex < timexAnns.Count && sentBegin <= BeginOffset(timexAnns[timexIndex]) && EndOffset(timexAnns[timexIndex]) <= sentEnd) { ++sublistEnd; ++timexIndex; } // set the sentence timexes sentence.Set(typeof(TimeAnnotations.TimexAnnotations), timexAnns.SubList(sublistBegin, sublistEnd)); } }
public static ProcessBuilder If(this ProcessBuilder builder, Func <bool> expression, DynamicIfStep truePart, DynamicIfStep falsePart) { var reference = Guid.Empty; return(builder.If(expression, truePart, falsePart, out reference)); }
private void SetProcessEnvironmentVariables(IReadOnlyDictionary <string, string> environment, ProcessBuilder processBuilder) { foreach (KeyValuePair <string, string> kvp in environment) { if (kvp.Value != null) { var envPipData = new PipDataBuilder(m_context.StringTable); // Casing for paths is not stable as reported by BuildPrediction. So here we try to guess if the value // represents a path, and normalize it string value = kvp.Value; if (!string.IsNullOrEmpty(value) && AbsolutePath.TryCreate(PathTable, value, out var absolutePath)) { envPipData.Add(absolutePath); } else { envPipData.Add(value); } processBuilder.SetEnvironmentVariable( StringId.Create(m_context.StringTable, kvp.Key), envPipData.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping)); } } }
/// <summary> /// Adds a step to the process that executes a simple GET request /// </summary> /// <param name="url"></param> /// <returns></returns> public static ProcessBuilder AddStep(this ProcessBuilder builder, string url) { var reference = Guid.Empty; return(builder.AddStep(url, out reference)); }
/// <summary>Start the process defined by the ProcessBuilder, and run until complete.</summary> /// <remarks> /// Start the process defined by the ProcessBuilder, and run until complete. /// Process output and errors will be written to System.out and System.err, /// respectively. /// </remarks> /// <param name="builder">The ProcessBuilder defining the process to run.</param> public static void Run(ProcessBuilder builder) { Run(builder, null, null); }
/// <summary> /// Adds a step to the process that executes a simple GET request /// </summary> /// <param name="url"></param> /// <param name="reference"></param> /// <returns>Returns a reference code to the step created</returns> public static ProcessBuilder AddStep(this ProcessBuilder builder, string url, out Guid reference) { return(builder.AddStep(url, HttpMethod.Get, null, out reference)); }
/// <summary> /// Adds a step to the process that executes a request with the given method and data /// </summary> /// <param name="url"></param> /// <param name="method"></param> /// <param name="data"></param> /// <returns></returns> public static ProcessBuilder AddStep(this ProcessBuilder builder, string url, HttpMethod method, object data) { var reference = Guid.Empty; return(builder.AddStep(url, method, data, out reference)); }
public TagBuilder(string id, ProcessBuilder parent) { _parent = parent; Id = id; }
public void BothColumnsKeepNull() { var topic = TestExecuter.GetTopic(); var builder = new ProcessBuilder() { InputProcess = TestData.PersonalAssetsPivot(topic), Mutators = new MutatorList() { new UnpivotMutator(topic, null) { FixColumns = new List <ColumnCopyConfiguration>() { new ColumnCopyConfiguration("id", "assetId"), new ColumnCopyConfiguration("personName"), }, NewColumnForDimension = "asset-kind", NewColumnForValue = "amount", IgnoreIfValueIsNull = false, ValueColumns = new[] { "cars", "houses", "kids" }, }, }, }; var result = TestExecuter.Execute(builder); Assert.AreEqual(12, result.MutatedRows.Count); Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >() { new CaseInsensitiveStringKeyDictionary <object>() { ["assetId"] = 1, ["personName"] = "A", ["asset-kind"] = "cars", ["amount"] = 1 }, new CaseInsensitiveStringKeyDictionary <object>() { ["assetId"] = 1, ["personName"] = "A", ["asset-kind"] = "houses", ["amount"] = 1 }, new CaseInsensitiveStringKeyDictionary <object>() { ["assetId"] = 1, ["personName"] = "A", ["asset-kind"] = "kids", ["amount"] = 2 }, new CaseInsensitiveStringKeyDictionary <object>() { ["personName"] = "C", ["asset-kind"] = "cars", ["amount"] = 2 }, new CaseInsensitiveStringKeyDictionary <object>() { ["personName"] = "C", ["asset-kind"] = "houses", ["amount"] = 1 }, new CaseInsensitiveStringKeyDictionary <object>() { ["personName"] = "C", ["asset-kind"] = "kids", ["amount"] = 3 }, new CaseInsensitiveStringKeyDictionary <object>() { ["assetId"] = 3, ["personName"] = "D", ["asset-kind"] = "cars" }, new CaseInsensitiveStringKeyDictionary <object>() { ["assetId"] = 3, ["personName"] = "D", ["asset-kind"] = "houses", ["amount"] = 1 }, new CaseInsensitiveStringKeyDictionary <object>() { ["assetId"] = 3, ["personName"] = "D", ["asset-kind"] = "kids", ["amount"] = 3 }, new CaseInsensitiveStringKeyDictionary <object>() { ["assetId"] = 4, ["personName"] = "E", ["asset-kind"] = "cars", ["amount"] = "6" }, new CaseInsensitiveStringKeyDictionary <object>() { ["assetId"] = 4, ["personName"] = "E", ["asset-kind"] = "houses", ["amount"] = 1 }, new CaseInsensitiveStringKeyDictionary <object>() { ["assetId"] = 4, ["personName"] = "E", ["asset-kind"] = "kids", ["amount"] = 3 } }); var exceptions = topic.Context.GetExceptions(); Assert.AreEqual(0, exceptions.Count); }
private bool TryConfigureProcessBuilder(ProcessBuilder processBuilder, NinjaNode node, QualifierId qualifierId) { SeparateExecutableFromCommands(node.Command, out string executable, out string args); if (!TryFindExecutablePath(executable, out AbsolutePath exePath)) { Tracing.Logger.Log.InvalidExecutablePath(m_context.LoggingContext, Location.FromFile(m_specPath.ToString(m_context.PathTable)), node.Command); return(false); } FileArtifact prExeArtifact = FileArtifact.CreateSourceFile(exePath); processBuilder.Executable = prExeArtifact; processBuilder.AddInputFile(prExeArtifact); using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder()) { var pipDataBuilder = pipDataBuilderWrapper.Instance; pipDataBuilder.Add(args); processBuilder.ArgumentsBuilder.Add(pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping)); } if (node.ResponseFile.HasValue) { using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder()) { var pipDataBuilder = pipDataBuilderWrapper.Instance; pipDataBuilder.Add(node.ResponseFile?.Content); PipData responseFileData = pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping); // We tell the process builder to write the response file but to not add any arguments to the process (requiresExplicitArgument = false) // because that information is already in node.Command var rspFileSpec = ResponseFileSpecification.Builder() .ExplicitData(responseFileData) .RequiresArgument(false) .ExplicitPath((AbsolutePath)node.ResponseFile?.Path) .Build(); processBuilder.SetResponseFileSpecification(rspFileSpec); } } // TODO: Maybe a better description. Add ninja description or change command for input/outputs processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {node.Rule} - {executable} :: [{node.Command}]")); processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads | Process.Options.OutputsMustRemainWritable | Process.Options.OutputsMustRemainWritable; processBuilder.EnableTempDirectory(); // Working directory - the directory containing the ninja spec file // Ninja generators may express paths relative to this processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(m_specPath.GetParent(m_context.PathTable)); // Untrack directories UntrackFilesAndDirectories(processBuilder.WorkingDirectory, processBuilder); // Allow some surviving child process AddRequiredSurvivingChildren(processBuilder); // Environment variables SetEnvironmentVariables(processBuilder, node); return(true); }
public ScriptBuilder(ProcessBuilder processBuilder, TflScript script) { _processBuilder = processBuilder; _script = script; }
public ISequence Build() { return(ProcessBuilder.Build()); }