public void ComplexTestCombinedWithSerializer()
{
var topic = TestExecuter.GetTopic();
var builder = new ProcessBuilder()
{
InputProcess = TestData.Person(topic),
Mutators = new MutatorList()
{
new InPlaceConvertMutator(topic, null)
{
Columns = new[] { "birthDate" },
TypeConverter = new DateConverterAuto(new CultureInfo("hu-HU")),
ActionIfInvalid = InvalidValueAction.Throw,
},
new ExplodeMutator(topic, null)
{
RowCreator = (proc, row) =>
{
var newRow = new SlimRow
{
["personModel"] = new PersonModel()
{
Id = row.GetAs <int>("id"),
Name = row.GetAs <string>("name"),
Age = row.GetAs <int?>("age"),
BirthDate = row.GetAs <DateTime?>("birthDate"),
}
};
return(new[] { newRow });
},
},
new DataContractXmlSerializerMutator <PersonModel>(topic, "serialize to XML byte[]")
{
ColumnConfiguration = new ColumnCopyConfiguration("personModel", "personModelXml"),
},
new RemoveColumnMutator(topic, null)
{
Columns = new[] { "personModel" },
},
new InPlaceConvertMutator(topic, "deserialize from XML byte[]")
{
Columns = new[] { "personModelXml" },
TypeConverter = new DataContractXmlDeSerializerConverter <PersonModel>(),
},
new RenameColumnMutator(topic, null)
{
ColumnConfiguration = new List <ColumnRenameConfiguration>()
{
new ColumnRenameConfiguration("personModelXml", "personModel"),
},
},
new ExplodeMutator(topic, null)
{
RowCreator = (proc, row) =>
{
var personModel = row.GetAs <PersonModel>("personModel");
var newRow = new SlimRow()
{
["id"] = personModel.Id,
["name"] = personModel.Name,
["age"] = personModel.Age,
["birthDate"] = personModel.BirthDate,
};
return(new[] { newRow });
},
},
},
};
var result = TestExecuter.Execute(builder);
Assert.AreEqual(7, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 0, ["name"] = "A", ["age"] = 17, ["birthDate"] = new DateTime(2010, 12, 9, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 1, ["name"] = "B", ["age"] = 8, ["birthDate"] = new DateTime(2011, 2, 1, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 2, ["name"] = "C", ["age"] = 27, ["birthDate"] = new DateTime(2014, 1, 21, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 3, ["name"] = "D", ["age"] = 39, ["birthDate"] = new DateTime(2018, 7, 11, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 4, ["name"] = "E", ["age"] = -3
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 5, ["name"] = "A", ["age"] = 11, ["birthDate"] = new DateTime(2013, 5, 15, 0, 0, 0, 0)
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["id"] = 6, ["name"] = "fake", ["birthDate"] = new DateTime(2018, 1, 9, 0, 0, 0, 0)
}
});
var exceptions = topic.Context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
private bool TryConfigureProcessBuilder(
ProcessBuilder processBuilder,
PipConstructionHelper pipConstructionHelper,
ProjectWithPredictions project,
QualifierId qualifierId,
out AbsolutePath outputResultCacheFile,
out string failureDetail)
{
outputResultCacheFile = AbsolutePath.Invalid;
if (!TrySetBuildToolExecutor(pipConstructionHelper, processBuilder, project))
{
failureDetail = "Failed to construct tooldefinition";
return(false);
}
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.FullPath.GetParent(PathTable));
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// Run in a container if specified
if (m_resolverSettings.RunInContainer)
{
processBuilder.Options |= Process.Options.NeedsToRunInContainer;
processBuilder.ContainerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs;
}
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= m_resolverSettings.DoubleWritePolicy ?? DoubleWritePolicy.AllowSameContentDoubleWrites;
SetUntrackedFilesAndDirectories(processBuilder);
// Add the log directory and its corresponding files
var qualifier = m_context.QualifierTable.GetQualifier(qualifierId);
AbsolutePath logDirectory = GetLogDirectory(project, qualifier);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.log"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.wrn"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.err"), FileExistence.Optional);
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.prf"), FileExistence.Optional);
if (m_resolverSettings.EnableBinLogTracing == true)
{
processBuilder.AddOutputFile(logDirectory.Combine(PathTable, "msbuild.binlog"), FileExistence.Optional);
}
// Unless the legacy non-isolated mode is explicitly specified, the project builds in isolation, and therefore
// it produces an output cache file. This file is placed on the (unique) object directory for this project
if (m_resolverSettings.UseLegacyProjectIsolation != true)
{
var objectDirectory = pipConstructionHelper.GetUniqueObjectDirectory(project.FullPath.GetName(PathTable));
outputResultCacheFile = objectDirectory.Path.Combine(PathTable, PathAtom.Create(PathTable.StringTable, OutputCacheFileName));
processBuilder.AddOutputFile(outputResultCacheFile, FileExistence.Required);
}
// Path to the project
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(project.FullPath));
// Response file with the rest of the arguments
var rspFileSpec = ResponseFileSpecification.Builder()
.AllowForRemainingArguments(processBuilder.ArgumentsBuilder.CreateCursor())
.ForceCreation(true)
.Prefix("@")
.Build();
processBuilder.SetResponseFileSpecification(rspFileSpec);
if (!TryAddMsBuildArguments(project, processBuilder.ArgumentsBuilder, logDirectory, outputResultCacheFile, out failureDetail))
{
return(false);
}
// Q_SESSION_GUID is used to provide a unique build GUID to build tools and scripts.
// It'll cause full cache misses if we try to hash it as an input, however, so exclude.
processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, BuildEnvironmentConstants.QSessionGuidEnvVar));
// mspdbsrv: _MSPDBSRV_ENDPOINT_ sets up one mspdbsrv.exe instance per build target execution.
// However this process will live beyond the build.cmd or msbuild.exe call.
// Allow the pip job object to clean the process without complaint.
//
// vctip.exe: On any compile error this telemetry upload exe will be run as a detached process.
// Just let it be killed.
// TODO: Can we stop it running? https://stackoverflow.microsoft.com/questions/74425/how-to-disable-vctip-exe-in-vc14
//
// conhost.exe: This process needs a little bit more time to finish after the main process. We shouldn't be allowing
// this one to survive, we just need the timeout to be slightly more than zero. This will also be beneficial to other
// arbitrary processeses that need a little bit more time. But, apparently, setting a timeout has a perf impact that is
// being investigated. TODO: revisit this once this is fixed.
//
// All child processes: Don't wait to kill the processes.
// CODESYNC: CloudBuild repo TrackerExecutor.cs "info.NestedProcessTerminationTimeout = TimeSpan.Zero"
processBuilder.AllowedSurvivingChildProcessNames = ReadOnlyArray <PathAtom> .FromWithoutCopy(
PathAtom.Create(m_context.StringTable, "mspdbsrv.exe"),
PathAtom.Create(m_context.StringTable, "vctip.exe"),
PathAtom.Create(m_context.StringTable, "conhost.exe"),
PathAtom.Create(m_context.StringTable, "VBCSCompiler.exe"));
processBuilder.NestedProcessTerminationTimeout = TimeSpan.Zero;
SetProcessEnvironmentVariables(CreateEnvironment(logDirectory, project), processBuilder);
failureDetail = string.Empty;
return(true);
}
/// <summary> /// Gets IPC process builder. /// </summary> /// <returns></returns> public ProcessBuilder GetIpcProcessBuilder() => ProcessBuilder.CreateForTesting(Context.PathTable);
private bool TryScheduleIpcPip(Context context, ObjectLiteral obj, bool allowUndefinedTargetService, bool isServiceFinalization, out FileArtifact outputFile, out PipId pipId)
{
// IpcClientInfo
IIpcMoniker moniker = Converter.ExtractRef <IIpcMoniker>(obj, m_ipcSendMoniker, allowUndefined: false);
int? numRetries = Converter.ExtractNumber(obj, m_ipcSendMaxConnectRetries, allowUndefined: true);
int? retryDelayMillis = Converter.ExtractNumber(obj, m_ipcSendConnectRetryDelayMillis, allowUndefined: true);
var clientConfig = new ClientConfig(numRetries, retryDelayMillis);
var ipcClientInfo = new IpcClientInfo(moniker.ToStringId(context.StringTable), clientConfig);
// target service pip
PipId?servicePipId = Converter.ExtractValue <PipId>(obj, m_ipcSendTargetServicePip, allowUndefined: allowUndefinedTargetService);
// arguments
PipData arguments;
ReadOnlyArray <FileArtifact> fileDependencies;
ReadOnlyArray <DirectoryArtifact> directoryDependencies;
using (var ipcProcessBuilder = ProcessBuilder.Create(context.PathTable, context.FrontEndContext.GetPipDataBuilder()))
{
// process arguments
ArrayLiteral argumentsArrayLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendMessageBody);
TransformerExecuteArgumentsProcessor.ProcessArguments(context, ipcProcessBuilder, argumentsArrayLiteral);
// input file dependencies
var dependenciesArray = Converter.ExtractArrayLiteral(obj, m_ipcSendDependencies, allowUndefined: true);
if (dependenciesArray != null)
{
for (int i = 0; i < dependenciesArray.Length; i++)
{
ProcessImplicitDependency(ipcProcessBuilder, dependenciesArray[i], convContext: new ConversionContext(pos: i, objectCtx: dependenciesArray));
}
}
arguments = ipcProcessBuilder.ArgumentsBuilder.ToPipData(" ", PipDataFragmentEscaping.CRuntimeArgumentRules);
fileDependencies = ipcProcessBuilder.GetInputFilesSoFar();
directoryDependencies = ipcProcessBuilder.GetInputDirectoriesSoFar();
}
// output
AbsolutePath output = Converter.ExtractPath(obj, m_ipcSendOutputFile, allowUndefined: true);
if (!output.IsValid)
{
output = context.GetPipConstructionHelper().GetUniqueObjectDirectory(m_ipcObjectFolderName).Path.Combine(context.PathTable, m_ipcOutputFileName);
}
// tags
string[] tags = null;
var tagsArray = Converter.ExtractArrayLiteral(obj, m_executeTags, allowUndefined: true);
if (tagsArray != null && tagsArray.Count > 0)
{
tags = new string[tagsArray.Count];
for (int i = 0; i < tagsArray.Count; i++)
{
tags[i] = Converter.ExpectString(tagsArray[i], context: new ConversionContext(pos: i, objectCtx: tagsArray));
}
}
// skip materialization for files
FileOrDirectoryArtifact[] skipMaterializationArtifacts = CollectionUtilities.EmptyArray <FileOrDirectoryArtifact>();
ArrayLiteral skipMaterializationLiteral = Converter.ExtractArrayLiteral(obj, m_ipcSendLazilyMaterializedDependencies, allowUndefined: true);
if (skipMaterializationLiteral != null)
{
skipMaterializationArtifacts = new FileOrDirectoryArtifact[skipMaterializationLiteral.Length];
for (int i = 0; i < skipMaterializationLiteral.Length; i++)
{
Converter.ExpectFileOrStaticDirectory(
skipMaterializationLiteral[i],
out var fileArtifact,
out var staticDirectory,
context: new ConversionContext(pos: i, objectCtx: skipMaterializationLiteral));
Contract.Assert(fileArtifact.IsValid ^ staticDirectory != null);
skipMaterializationArtifacts[i] = fileArtifact.IsValid
? FileOrDirectoryArtifact.Create(fileArtifact)
: FileOrDirectoryArtifact.Create(staticDirectory.Root);
}
}
// must run on master
var mustRunOnMaster = Converter.ExtractOptionalBoolean(obj, m_ipcSendMustRunOnMaster) == true;
outputFile = FileArtifact.CreateOutputFile(output);
// create IPC pip and add it to the graph
return(context.GetPipConstructionHelper().TryAddIpc(
ipcClientInfo,
arguments,
outputFile,
servicePipDependencies: servicePipId != null ? ReadOnlyArray <PipId> .From(new[] { servicePipId.Value }) : ReadOnlyArray <PipId> .Empty,
fileDependencies: fileDependencies,
directoryDependencies: directoryDependencies,
skipMaterializationFor: ReadOnlyArray <FileOrDirectoryArtifact> .FromWithoutCopy(skipMaterializationArtifacts),
isServiceFinalization: isServiceFinalization,
mustRunOnMaster: mustRunOnMaster,
tags: tags,
out pipId));
}
/// <summary>
/// Adds all predicted dependencies as inputs, plus all individual inputs predicted for the project
/// </summary>
/// <remarks>
/// Adding all predicted dependencies is key to get the right scheduling. On the other hand, all predicted inputs
/// are not really needed since we are running in undeclared read mode. However, they contribute to make the weak fingerprint stronger (that
/// otherwise will likely be just a shared opaque output at the root).
/// </remarks>
private void ProcessInputs(
ProjectWithPredictions project,
ProcessBuilder processBuilder)
{
// Predicted output directories for all direct dependencies, plus the output directories for the given project itself
var knownOutputDirectories = project.ProjectReferences.SelectMany(reference => reference.PredictedOutputFolders).Union(project.PredictedOutputFolders);
// Add all predicted inputs that are recognized as true source files
// This is done to make the weak fingerprint stronger. Pips are scheduled so undeclared source reads are allowed. This means
// we don't actually need accurate (or in fact any) input predictions to run successfully. But we are trying to avoid the degenerate case
// of a very small weak fingerprint with too many candidates, that can slow down two-phase cache look-up.
foreach (AbsolutePath buildInput in project.PredictedInputFiles)
{
// If any of the predicted inputs is under the predicted output folder of a dependency, then there is a very good chance the predicted input is actually an intermediate file
// In that case, don't add the input as a source file to stay on the safe side. Otherwise we will have a file that is both declared as a source file and contained in a directory
// dependency.
if (knownOutputDirectories.Any(outputFolder => buildInput.IsWithin(PathTable, outputFolder)))
{
continue;
}
// If any of the predicted inputs is under an untracked directory scope, don't add it as an input
if (processBuilder.GetUntrackedDirectoryScopesSoFar().Any(untrackedDirectory => buildInput.IsWithin(PathTable, untrackedDirectory)))
{
continue;
}
processBuilder.AddInputFile(FileArtifact.CreateSourceFile(buildInput));
}
IEnumerable <ProjectWithPredictions> references;
// The default for EnableTransitiveProjectReferences is false, so it has to be true explicitly to kick in
if (m_resolverSettings.EnableTransitiveProjectReferences == true)
{
// In this case all the transitive closure is automatically exposed to the project as direct references
var transitiveReferences = new HashSet <ProjectWithPredictions>();
ComputeTransitiveDependenciesFor(project, transitiveReferences);
references = transitiveReferences;
}
else
{
// Only direct dependencies are declared.
// Add all known explicit inputs from project references. But rule out
// projects that have a known empty list of targets: those projects are not scheduled, so
// there is nothing to consume from them.
references = project.ProjectReferences.Where(projectReference => !projectReference.PredictedTargetsToExecute.TargetsAreKnownToBeEmpty);
}
var argumentsBuilder = processBuilder.ArgumentsBuilder;
foreach (ProjectWithPredictions projectReference in references)
{
bool outputsPresent = m_processOutputsPerProject.TryGetValue(projectReference, out MSBuildProjectOutputs projectOutputs);
if (!outputsPresent)
{
Contract.Assert(false, $"Pips must have been presented in dependency order: {projectReference.FullPath.ToString(PathTable)} missing, dependency of {project.FullPath.ToString(PathTable)}");
}
// Add all known output directories
foreach (StaticDirectory output in projectOutputs.OutputDirectories)
{
processBuilder.AddInputDirectory(output.Root);
}
// If the dependency was built in isolation, this project needs to access the generated cache files
if (projectOutputs.BuildsInIsolation)
{
var outputCache = projectOutputs.OutputCacheFile;
processBuilder.AddInputFile(outputCache);
// Instruct MSBuild to use the cache file from the associated dependency as an input.
// Flag /irc is the short form of /inputResultsCaches, and part of MSBuild 'build in isolation' mode.
using (argumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, string.Empty))
{
argumentsBuilder.Add(PipDataAtom.FromString("/irc:"));
argumentsBuilder.Add(PipDataAtom.FromAbsolutePath(outputCache));
}
}
}
}
private void ButtonCreateMetadata_Click(object sender, EventArgs e)
{
ProcessBuilder.EditTransition(GetTransition());
MessageBox.Show("Process metadata created successfully.");
}
private bool TryExecuteArgumentsToPipBuilder(
ProjectWithPredictions project,
QualifierId qualifierId,
out string failureDetail,
out Process scheduledProcess)
{
// We create a pip construction helper for each project
var pipConstructionHelper = GetPipConstructionHelperForProject(project, qualifierId);
using (var processBuilder = ProcessBuilder.Create(PathTable, m_context.GetPipDataBuilder()))
{
// Configure the process to add an assortment of settings: arguments, response file, etc.
if (!TryConfigureProcessBuilder(processBuilder, pipConstructionHelper, project, qualifierId, out AbsolutePath outputResultCacheFile, out failureDetail))
{
scheduledProcess = null;
return(false);
}
// Process all predicted outputs and inputs, including the predicted project dependencies
ProcessOutputs(project, processBuilder);
ProcessInputs(project, processBuilder);
// Try to schedule the process pip
if (!pipConstructionHelper.TryAddProcess(processBuilder, out ProcessOutputs outputs, out scheduledProcess))
{
failureDetail = "Failed to schedule the pip";
return(false);
}
// Add the computed outputs for this project, so dependencies can consume it
var outputDirectories = outputs.GetOutputDirectories();
// A valid output cache path indicates that the project is building in isolation
MSBuildProjectOutputs projectOutputs;
if (outputResultCacheFile == AbsolutePath.Invalid)
{
projectOutputs = MSBuildProjectOutputs.CreateLegacy(outputDirectories);
}
else
{
var success = outputs.TryGetOutputFile(outputResultCacheFile, out FileArtifact cacheFileArtifact);
if (!success)
{
Contract.Assert(false, I($"The output cache file {outputResultCacheFile.ToString(PathTable)} should be part of the project {project.FullPath.ToString(PathTable)} outputs."));
}
projectOutputs = MSBuildProjectOutputs.CreateIsolated(outputDirectories, cacheFileArtifact);
}
// If the project is not implementing the target protocol, emit corresponding warn/verbose
if (!project.ImplementsTargetProtocol)
{
if (project.ProjectReferences.Count != 0)
{
// Let's warn about this. Targets of the referenced projects may not be accurate
Tracing.Logger.Log.ProjectIsNotSpecifyingTheProjectReferenceProtocol(
m_context.LoggingContext,
Location.FromFile(project.FullPath.ToString(PathTable)),
project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable));
}
else
{
// Just a verbose message in this case
Tracing.Logger.Log.LeafProjectIsNotSpecifyingTheProjectReferenceProtocol(
m_context.LoggingContext,
Location.FromFile(project.FullPath.ToString(PathTable)),
project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable));
}
}
// Warn if default targets were appended to the targets to execute
if (project.PredictedTargetsToExecute.IsDefaultTargetsAppended)
{
Tracing.Logger.Log.ProjectPredictedTargetsAlsoContainDefaultTargets(
m_context.LoggingContext,
Location.FromFile(project.FullPath.ToString(PathTable)),
project.FullPath.GetName(m_context.PathTable).ToString(m_context.StringTable),
$"[{string.Join(";", project.PredictedTargetsToExecute.AppendedDefaultTargets)}]");
}
m_processOutputsPerProject[project] = projectOutputs;
failureDetail = string.Empty;
return(true);
}
}
private void CreateProcessFromControls()
{
ProcessBuilder.BuildProcess(Guid.NewGuid().ToString(), TextEditProcessName.Text, TextEditProcessCaption.Text, 2 /* WithoutState */);
}
private void ConfigureProcessBuilder(
ProcessBuilder processBuilder,
RushProject project)
{
SetCmdTool(processBuilder, project);
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder);
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient
// to avoid a large number of path sets
processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation;
// Try to preserve path set casing since many JavaScript projects deal with paths in a case-sensitive way
// Otherwise in Windows we force path sets to be all uppercase
processBuilder.Options |= Process.Options.PreservePathSetCasing;
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites;
// Untrack the user profile. The corresponding mount is already configured for not tracking source files, and with allowed undeclared source reads,
// any attempt to read into the user profile will fail to compute its corresponding hash
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile)));
// If dependencies are tracked with the shrinkwrap-deps file, then untrack everything under the Rush common temp folder, where all package
// dependencies are placed
if (m_resolverSettings.TrackDependenciesWithShrinkwrapDepsFile == true)
{
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(m_rushConfiguration.CommonTempFolder));
}
// Add the associated build script name as a tag, so filtering on 'build' or 'test' can happen
processBuilder.Tags = ReadOnlyArray <StringId> .FromWithoutCopy(new[] { StringId.Create(m_context.StringTable, project.ScriptCommandName) });
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
var logDirectory = GetLogDirectory(project);
processBuilder.SetStandardOutputFile(logDirectory.Combine(m_context.PathTable, "build.log"));
processBuilder.SetStandardErrorFile(logDirectory.Combine(m_context.PathTable, "error.log"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, " "))
{
// Execute the command and redirect the output to a designated log file
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.ScriptCommand));
// If we need to append arguments to the script command, do it here
if (m_customCommands.TryGetValue(project.ScriptCommandName, out IReadOnlyList <RushArgument> extraArguments))
{
foreach (RushArgument value in extraArguments)
{
AddRushArgumentToBuilder(processBuilder.ArgumentsBuilder, value);
}
}
}
}
FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable);
}
public override void execute(ActivityExecution execution)
{
readFields(execution);
IList <string> argList = new List <string>();
argList.Add(commandStr);
if (!string.ReferenceEquals(arg1Str, null))
{
argList.Add(arg1Str);
}
if (!string.ReferenceEquals(arg2Str, null))
{
argList.Add(arg2Str);
}
if (!string.ReferenceEquals(arg3Str, null))
{
argList.Add(arg3Str);
}
if (!string.ReferenceEquals(arg4Str, null))
{
argList.Add(arg4Str);
}
if (!string.ReferenceEquals(arg5Str, null))
{
argList.Add(arg5Str);
}
ProcessBuilder processBuilder = new ProcessBuilder(argList);
try
{
processBuilder.redirectErrorStream(redirectErrorFlag);
if (cleanEnvBoolan.Value)
{
IDictionary <string, string> env = processBuilder.environment();
env.Clear();
}
if (!string.ReferenceEquals(directoryStr, null) && directoryStr.Length > 0)
{
processBuilder.directory(new File(directoryStr));
}
Process process = processBuilder.start();
if (waitFlag.Value)
{
int errorCode = process.waitFor();
if (!string.ReferenceEquals(resultVariableStr, null))
{
string result = convertStreamToStr(process.InputStream);
execution.setVariable(resultVariableStr, result);
}
if (!string.ReferenceEquals(errorCodeVariableStr, null))
{
execution.setVariable(errorCodeVariableStr, Convert.ToString(errorCode));
}
}
}
catch (Exception e)
{
throw LOG.shellExecutionException(e);
}
leave(execution);
}
/// <summary>
/// fork ourselves in a new jvm. sets -Dtests.crashmode=true </summary>
public virtual void ForkTest()
{
IList<string> cmd = new List<string>();
cmd.Add(System.getProperty("java.home") + System.getProperty("file.separator") + "bin" + System.getProperty("file.separator") + "java");
cmd.Add("-Xmx512m");
cmd.Add("-Dtests.crashmode=true");
// passing NIGHTLY to this test makes it run for much longer, easier to catch it in the act...
cmd.Add("-Dtests.nightly=true");
cmd.Add("-DtempDir=" + TempDir.Path);
cmd.Add("-Dtests.seed=" + SeedUtils.formatSeed(Random().NextLong()));
cmd.Add("-ea");
cmd.Add("-cp");
cmd.Add(System.getProperty("java.class.path"));
cmd.Add("org.junit.runner.JUnitCore");
cmd.Add(this.GetType().Name);
ProcessBuilder pb = new ProcessBuilder(cmd);
pb.directory(TempDir);
pb.redirectErrorStream(true);
Process p = pb.Start();
// We pump everything to stderr.
PrintStream childOut = System.err;
Thread stdoutPumper = ThreadPumper.Start(p.InputStream, childOut);
Thread stderrPumper = ThreadPumper.Start(p.ErrorStream, childOut);
if (VERBOSE)
{
childOut.println(">>> Begin subprocess output");
}
p.waitFor();
stdoutPumper.Join();
stderrPumper.Join();
if (VERBOSE)
{
childOut.println("<<< End subprocess output");
}
}
public SearchTypeBuilder(ProcessBuilder processBuilder, TflSearchType searchType) {
_processBuilder = processBuilder;
_searchType = searchType;
}
/// <summary>
/// Forces every step of the process to use test options
/// </summary>
/// <param name="builder"></param>
/// <returns></returns>
public static ProcessBuilder ForceUseTestOptions(this ProcessBuilder builder)
{
builder.StepOptions = StepOptions.Test;
return(builder);
}
public static Process start(this ProcessBuilder processBuilder)
{
return(processBuilder.Start());
}
public MapBuilder(ProcessBuilder processBuilder, TflMap map)
{
_processBuilder = processBuilder;
_map = map;
}
public Process startInterpreter(String pythonExe)
{
return null;
#if NYI
Process p;
try
{
ProcessBuilder builder = new ProcessBuilder(pythonExe, "-i", jsonizer);
builder.redirectErrorStream(true);
builder.redirectError(parserLog));
builder.redirectOutput(parserLog));
builder.environment().remove("PYTHONPATH");
p = builder.start();
}
catch (Exception e)
{
_.msg("Failed to start: " + pythonExe);
return null;
}
return p;
#endif
}
public RelationshipBuilder(ProcessBuilder processBuilder, TflRelationship relationship) {
_processBuilder = processBuilder;
_relationship = relationship;
}
public void Test_DynamicActivities()
{
var process = new ProcessBuilder()
.CreateActivity("act1")
.CreateExecution("act1", _ =>
{
return((c, t) =>
{
// 获取需要动态分拆的数量
int number = t.Get("itemcount", 1);
Transition nextTranistion = t.Destination.OutgoingTransitions?[0];
if (nextTranistion != null)
{
Transition dynamicTransition = null;
// 第一个item使用原有的transition
// 从第二个开始，添加transition并把
for (int i = 2; i <= number; i++)
{
dynamicTransition = new Transition
{
Source = nextTranistion.Source,
Destination = nextTranistion.Destination
};
nextTranistion.Source.AddOutgoingTransition(dynamicTransition);
nextTranistion.Destination.AddIncomingTransition(dynamicTransition);
}
}
return Task.CompletedTask;
});
})
.CreateActivity("act2")
.CreateExecution("act2", _ =>
{
return((c, t) =>
{
Console.WriteLine(c.Get("accepted", 0));
c.Set("accepted", c.Get("accepted", 0) + 1);
return Task.CompletedTask;
});
})
.CreateTransition("act1", "act2")
.CreateActivity("act3")
.CreateTransition("act2", "act3")
.CreateActivity("act4")
.CreateTransition("act3", "act4")
.CreateActivity("act5")
.CreateTransition("act4", "act5")
.CreateExecution("act5", _ =>
{
return((c, t) =>
{
// TransitionState[] availableStates = new TransitionState[] {
// TransitionState.Passed, TransitionState.Blocked
// };
// if (t.Destination.IncomingTransitions.Any(ts => availableStates.Contains(ts.State) == false))
// {
// t.CurrentTransition.SetState(TransitionState.Blocked);
// }
int number = c.Get("itemcount", 1);
int accepted = c.Get("accepted", 0);
if (number > accepted)
{
t.CurrentTransition.SetState(TransitionState.Blocked);
}
return Task.CompletedTask;
});
})
.SetStart("act1")
.Build();
var initData = new Dictionary <string, object>();
int itemcount = 5;
initData["itemcount"] = itemcount;
process.Start(initData);
Assert.Equal(itemcount, process.Context.Get("accepted", 0));
}
private void LaunchGame(bool online)
{
Log.Info("Launching game...");
if (this.CheckCurrentProfile())
{
if (online && this.Profiles.CurrentProfile != 0)
{
Messages.Show(this.Window, "CantPlayOnline", "Impossible", MessageBoxButton.OK, MessageBoxImage.Error);
this.UiManager.Working = false;
this.UiManager.ButtonsEnabled = true;
this.UiManager.CanPlayOnline = this.UiManager.SelectedProfile == 0;
}
else
{
ProcessBuilder builder = new ProcessBuilder();
builder.WorkingDirectory = this.GtaPath;
if (this.Settings.UseRph && File.Exists(Path.Combine(this.GtaPath, "RAGEPluginHook.exe")))
{
Log.Info("Starting RAGE Plugin Hook process...");
builder.FilePath = Path.Combine(this.GtaPath, "RAGEPluginHook.exe");
}
else if (this.IsSteamVersion())
{
Log.Info("Starting steam game process...");
builder.FilePath = Path.Combine(this.SteamPath, "steam.exe");
builder.AddArgument("-applaunch 271590");
}
else
{
Log.Info("Starting game process...");
builder.FilePath = Path.Combine(this.GtaPath, "GTAVLauncher.exe");
}
Log.Info("Setting game language to " + this.Settings.GtaLanguage);
builder.AddArgument("-uilanguage " + this.Settings.GtaLanguage);
if (online)
{
builder.AddArgument("-StraightIntoFreemode");
}
else if (this.Profiles.CurrentProfile != 0 && this.Settings.OfflineMode)
{
builder.AddArgument("-scOfflineOnly");
}
Log.Info("Executing " + builder);
builder.StartProcess();
this.CloseLauncher();
}
}
else
{
Log.Info("Aborting game launch.");
this.UiManager.Working = false;
this.UiManager.ButtonsEnabled = true;
this.UiManager.CanPlayOnline = this.UiManager.SelectedProfile == 0;
}
}
/// <summary>
/// Adds a step to the process that executes a request with a lazy loaded request data
/// </summary>
/// <param name="url"></param>
/// <param name="method"></param>
/// <param name="lazyData"></param>
/// <returns></returns>
public static ProcessBuilder AddStep(this ProcessBuilder builder, string url, HttpMethod method, Func <object> lazyData, StepOptions options = null)
{
var reference = Guid.Empty;
return(builder.AddStep(url, method, lazyData, out reference, options));
}
public SearchTypeBuilder(ProcessBuilder processBuilder, TflSearchType searchType)
{
_processBuilder = processBuilder;
_searchType = searchType;
}
public static DynamicStep GetDynamicStep(this ProcessBuilder builder, Guid reference)
{
return(builder.Steps.OfType <DynamicStep>().FirstOrDefault(s => s.Reference == reference));
}
/// <exception cref="System.IO.IOException"/>
public virtual void Annotate(ICoreMap document)
{
// write input file in GUTime format
IElement inputXML = ToInputXML(document);
File inputFile = File.CreateTempFile("gutime", ".input");
//Document doc = new Document(inputXML);
PrintWriter inputWriter = new PrintWriter(inputFile);
inputWriter.Println(XMLUtils.NodeToString(inputXML, false));
// new XMLOutputter().output(inputXML, inputWriter);
inputWriter.Close();
bool useFirstDate = (!document.ContainsKey(typeof(CoreAnnotations.CalendarAnnotation)) && !document.ContainsKey(typeof(CoreAnnotations.DocDateAnnotation)));
List <string> args = new List <string>();
args.Add("perl");
args.Add("-I" + this.gutimePath.GetPath());
args.Add(new File(this.gutimePath, "TimeTag.pl").GetPath());
if (useFirstDate)
{
args.Add("-FDNW");
}
args.Add(inputFile.GetPath());
// run GUTime on the input file
ProcessBuilder process = new ProcessBuilder(args);
StringWriter outputWriter = new StringWriter();
SystemUtils.Run(process, outputWriter, null);
string output = outputWriter.GetBuffer().ToString();
Pattern docClose = Pattern.Compile("</DOC>.*", Pattern.Dotall);
output = docClose.Matcher(output).ReplaceAll("</DOC>");
//The TimeTag.pl result file contains next tags which must be removed
output = output.ReplaceAll("<lex.*?>", string.Empty);
output = output.Replace("</lex>", string.Empty);
output = output.Replace("<NG>", string.Empty);
output = output.Replace("</NG>", string.Empty);
output = output.Replace("<VG>", string.Empty);
output = output.Replace("</VG>", string.Empty);
output = output.Replace("<s>", string.Empty);
output = output.Replace("</s>", string.Empty);
// parse the GUTime output
IElement outputXML;
try
{
outputXML = XMLUtils.ParseElement(output);
}
catch (Exception ex)
{
throw new Exception(string.Format("error:\n%s\ninput:\n%s\noutput:\n%s", ex, IOUtils.SlurpFile(inputFile), output), ex);
}
/*
* try {
* outputXML = new SAXBuilder().build(new StringReader(output)).getRootElement();
* } catch (JDOMException e) {
* throw new RuntimeException(String.format("error:\n%s\ninput:\n%s\noutput:\n%s",
* e, IOUtils.slurpFile(inputFile), output));
* } */
inputFile.Delete();
// get Timex annotations
IList <ICoreMap> timexAnns = ToTimexCoreMaps(outputXML, document);
document.Set(typeof(TimeAnnotations.TimexAnnotations), timexAnns);
if (outputResults)
{
System.Console.Out.WriteLine(timexAnns);
}
// align Timex annotations to sentences
int timexIndex = 0;
foreach (ICoreMap sentence in document.Get(typeof(CoreAnnotations.SentencesAnnotation)))
{
int sentBegin = BeginOffset(sentence);
int sentEnd = EndOffset(sentence);
// skip times before the sentence
while (timexIndex < timexAnns.Count && BeginOffset(timexAnns[timexIndex]) < sentBegin)
{
++timexIndex;
}
// determine times within the sentence
int sublistBegin = timexIndex;
int sublistEnd = timexIndex;
while (timexIndex < timexAnns.Count && sentBegin <= BeginOffset(timexAnns[timexIndex]) && EndOffset(timexAnns[timexIndex]) <= sentEnd)
{
++sublistEnd;
++timexIndex;
}
// set the sentence timexes
sentence.Set(typeof(TimeAnnotations.TimexAnnotations), timexAnns.SubList(sublistBegin, sublistEnd));
}
}
public static ProcessBuilder If(this ProcessBuilder builder, Func <bool> expression, DynamicIfStep truePart, DynamicIfStep falsePart)
{
var reference = Guid.Empty;
return(builder.If(expression, truePart, falsePart, out reference));
}
private void SetProcessEnvironmentVariables(IReadOnlyDictionary <string, string> environment, ProcessBuilder processBuilder)
{
foreach (KeyValuePair <string, string> kvp in environment)
{
if (kvp.Value != null)
{
var envPipData = new PipDataBuilder(m_context.StringTable);
// Casing for paths is not stable as reported by BuildPrediction. So here we try to guess if the value
// represents a path, and normalize it
string value = kvp.Value;
if (!string.IsNullOrEmpty(value) && AbsolutePath.TryCreate(PathTable, value, out var absolutePath))
{
envPipData.Add(absolutePath);
}
else
{
envPipData.Add(value);
}
processBuilder.SetEnvironmentVariable(
StringId.Create(m_context.StringTable, kvp.Key),
envPipData.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
}
}
}
/// <summary>
/// Adds a step to the process that executes a simple GET request
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public static ProcessBuilder AddStep(this ProcessBuilder builder, string url)
{
var reference = Guid.Empty;
return(builder.AddStep(url, out reference));
}
/// <summary>Start the process defined by the ProcessBuilder, and run until complete.</summary>
/// <remarks>
/// Start the process defined by the ProcessBuilder, and run until complete.
/// Process output and errors will be written to System.out and System.err,
/// respectively.
/// </remarks>
/// <param name="builder">The ProcessBuilder defining the process to run.</param>
public static void Run(ProcessBuilder builder)
{
Run(builder, null, null);
}
/// <summary>
/// Adds a step to the process that executes a simple GET request
/// </summary>
/// <param name="url"></param>
/// <param name="reference"></param>
/// <returns>Returns a reference code to the step created</returns>
public static ProcessBuilder AddStep(this ProcessBuilder builder, string url, out Guid reference)
{
return(builder.AddStep(url, HttpMethod.Get, null, out reference));
}
public RelationshipBuilder(ProcessBuilder processBuilder, TflRelationship relationship)
{
_processBuilder = processBuilder;
_relationship = relationship;
}
/// <summary>
/// Adds a step to the process that executes a request with the given method and data
/// </summary>
/// <param name="url"></param>
/// <param name="method"></param>
/// <param name="data"></param>
/// <returns></returns>
public static ProcessBuilder AddStep(this ProcessBuilder builder, string url, HttpMethod method, object data)
{
var reference = Guid.Empty;
return(builder.AddStep(url, method, data, out reference));
}
public TagBuilder(string id, ProcessBuilder parent)
{
_parent = parent;
Id = id;
}
public MapBuilder(ProcessBuilder processBuilder, TflMap map) {
_processBuilder = processBuilder;
_map = map;
}
public void BothColumnsKeepNull()
{
var topic = TestExecuter.GetTopic();
var builder = new ProcessBuilder()
{
InputProcess = TestData.PersonalAssetsPivot(topic),
Mutators = new MutatorList()
{
new UnpivotMutator(topic, null)
{
FixColumns = new List <ColumnCopyConfiguration>()
{
new ColumnCopyConfiguration("id", "assetId"),
new ColumnCopyConfiguration("personName"),
},
NewColumnForDimension = "asset-kind",
NewColumnForValue = "amount",
IgnoreIfValueIsNull = false,
ValueColumns = new[] { "cars", "houses", "kids" },
},
},
};
var result = TestExecuter.Execute(builder);
Assert.AreEqual(12, result.MutatedRows.Count);
Assert.That.ExactMatch(result.MutatedRows, new List <CaseInsensitiveStringKeyDictionary <object> >()
{
new CaseInsensitiveStringKeyDictionary <object>()
{
["assetId"] = 1, ["personName"] = "A", ["asset-kind"] = "cars", ["amount"] = 1
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["assetId"] = 1, ["personName"] = "A", ["asset-kind"] = "houses", ["amount"] = 1
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["assetId"] = 1, ["personName"] = "A", ["asset-kind"] = "kids", ["amount"] = 2
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["personName"] = "C", ["asset-kind"] = "cars", ["amount"] = 2
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["personName"] = "C", ["asset-kind"] = "houses", ["amount"] = 1
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["personName"] = "C", ["asset-kind"] = "kids", ["amount"] = 3
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["assetId"] = 3, ["personName"] = "D", ["asset-kind"] = "cars"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["assetId"] = 3, ["personName"] = "D", ["asset-kind"] = "houses", ["amount"] = 1
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["assetId"] = 3, ["personName"] = "D", ["asset-kind"] = "kids", ["amount"] = 3
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["assetId"] = 4, ["personName"] = "E", ["asset-kind"] = "cars", ["amount"] = "6"
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["assetId"] = 4, ["personName"] = "E", ["asset-kind"] = "houses", ["amount"] = 1
},
new CaseInsensitiveStringKeyDictionary <object>()
{
["assetId"] = 4, ["personName"] = "E", ["asset-kind"] = "kids", ["amount"] = 3
}
});
var exceptions = topic.Context.GetExceptions();
Assert.AreEqual(0, exceptions.Count);
}
private bool TryConfigureProcessBuilder(ProcessBuilder processBuilder, NinjaNode node, QualifierId qualifierId)
{
SeparateExecutableFromCommands(node.Command, out string executable, out string args);
if (!TryFindExecutablePath(executable, out AbsolutePath exePath))
{
Tracing.Logger.Log.InvalidExecutablePath(m_context.LoggingContext, Location.FromFile(m_specPath.ToString(m_context.PathTable)), node.Command);
return(false);
}
FileArtifact prExeArtifact = FileArtifact.CreateSourceFile(exePath);
processBuilder.Executable = prExeArtifact;
processBuilder.AddInputFile(prExeArtifact);
using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder())
{
var pipDataBuilder = pipDataBuilderWrapper.Instance;
pipDataBuilder.Add(args);
processBuilder.ArgumentsBuilder.Add(pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
}
if (node.ResponseFile.HasValue)
{
using (var pipDataBuilderWrapper = m_context.GetPipDataBuilder())
{
var pipDataBuilder = pipDataBuilderWrapper.Instance;
pipDataBuilder.Add(node.ResponseFile?.Content);
PipData responseFileData = pipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping);
// We tell the process builder to write the response file but to not add any arguments to the process (requiresExplicitArgument = false)
// because that information is already in node.Command
var rspFileSpec = ResponseFileSpecification.Builder()
.ExplicitData(responseFileData)
.RequiresArgument(false)
.ExplicitPath((AbsolutePath)node.ResponseFile?.Path)
.Build();
processBuilder.SetResponseFileSpecification(rspFileSpec);
}
}
// TODO: Maybe a better description. Add ninja description or change command for input/outputs
processBuilder.ToolDescription = StringId.Create(m_context.StringTable,
I($"{m_moduleDefinition.Descriptor.Name} - {node.Rule} - {executable} :: [{node.Command}]"));
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads | Process.Options.OutputsMustRemainWritable | Process.Options.OutputsMustRemainWritable;
processBuilder.EnableTempDirectory();
// Working directory - the directory containing the ninja spec file
// Ninja generators may express paths relative to this
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(m_specPath.GetParent(m_context.PathTable));
// Untrack directories
UntrackFilesAndDirectories(processBuilder.WorkingDirectory, processBuilder);
// Allow some surviving child process
AddRequiredSurvivingChildren(processBuilder);
// Environment variables
SetEnvironmentVariables(processBuilder, node);
return(true);
}
public ScriptBuilder(ProcessBuilder processBuilder, TflScript script) {
_processBuilder = processBuilder;
_script = script;
}
public ISequence Build()
{
return(ProcessBuilder.Build());
}
