private FullSymbol GetFullSymbolFromProject(ProjectWithPredictions project)
{
// We construct the name of the value using the project name and its global properties
// Observe this symbol has to be unique wrt another symbol coming from the same physical project (i.e. same project full
// path) but different global properties. The project full path is already being passed as part of the 'key' when creating the
// pip construction helper
var valueName = PipConstructionUtilities.SanitizeStringForSymbol(project.FullPath.GetName(PathTable).ToString(m_context.StringTable));
// If global properties are present, we append to the value name a flatten representation of them
// There should always be a 'IsGraphBuild' property, so we count > 1
Contract.Assert(project.GlobalProperties.ContainsKey(s_isGraphBuildProperty));
if (project.GlobalProperties.Count > 1)
{
valueName += ".";
}
valueName += string.Join(
".",
project.GlobalProperties
// let's sort global properties keys to make sure the same string is generated consistently
// case-sensitivity is already handled (and ignored) by GlobalProperties class
.Where(kvp => kvp.Key != s_isGraphBuildProperty)
.OrderBy(kvp => kvp.Key, StringComparer.Ordinal)
.Select(gp => $"{PipConstructionUtilities.SanitizeStringForSymbol(gp.Key)}_{PipConstructionUtilities.SanitizeStringForSymbol(gp.Value)}"));
var fullSymbol = FullSymbol.Create(m_context.SymbolTable, valueName);
return(fullSymbol);
}
private void ConfigureProcessBuilder(
ProcessBuilder processBuilder,
RushProject project)
{
SetCmdTool(processBuilder, project);
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder);
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient
// to avoid a large number of path sets
processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation;
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites;
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
var logDirectory = GetLogDirectory(project);
var logFile = logDirectory.Combine(PathTable, "build.log");
// Execute the build command and redirect the output to a designated log file
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C"));
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.BuildCommand));
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(">"));
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(logFile));
processBuilder.AddOutputFile(logFile, FileExistence.Required);
FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable);
}
private AbsolutePath GetLogDirectory(ProjectWithPredictions projectFile, Qualifier qualifier)
{
var success = Root.TryGetRelative(PathTable, projectFile.FullPath, out var inFolderPathFromEnlistmentRoot);
Contract.Assert(success);
// We hardcode the log to go under the output directory Logs/MSBuild (and follow the project structure underneath)
// The 'official' log directory (defined by Configuration.Logging) is not stable in CloudBuild across machines, and therefore it would
// introduce cache misses
var result = m_frontEndHost.Configuration.Layout.OutputDirectory
.Combine(PathTable, "Logs")
.Combine(PathTable, "MSBuild")
.Combine(PathTable, inFolderPathFromEnlistmentRoot);
// Build a string with just the qualifier and global property values (e.g. 'debug-x86'). That should be unique enough.
// Projects can be evaluated multiple times with different global properties (but same qualifiers), so just
// the qualifier name is not enough
List <string> values = qualifier.Values.Select(value => value.ToString(m_context.StringTable))
.Union(projectFile.GlobalProperties.Where(kvp => kvp.Key != s_isGraphBuildProperty).Select(kvp => kvp.Value))
.Select(value => PipConstructionUtilities.SanitizeStringForSymbol(value))
.OrderBy(value => value, StringComparer.Ordinal) // Let's make sure we always produce the same string for the same set of values
.ToList();
if (values.Count > 0)
{
var valueIdentifier = string.Join("-", values);
result = result.Combine(PathTable, valueIdentifier);
}
return(result);
}
/// <summary>
/// Configures the process builder to execute the specified commands
/// </summary>
protected virtual void ConfigureProcessBuilder(
ProcessBuilder processBuilder,
JavaScriptProject project)
{
SetCmdTool(processBuilder, project);
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder);
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient
// to avoid a large number of path sets
processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation;
// Try to preserve path set casing since many JavaScript projects deal with paths in a case-sensitive way
// Otherwise in Windows we force path sets to be all uppercase
processBuilder.Options |= Process.Options.PreservePathSetCasing;
// By default the double write policy is to allow same content double writes.
processBuilder.DoubleWritePolicy |= DoubleWritePolicy.AllowSameContentDoubleWrites;
// Untrack the user profile. The corresponding mount is already configured for not tracking source files, and with allowed undeclared source reads,
// any attempt to read into the user profile will fail to compute its corresponding hash
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile)));
// Add the associated build script name as a tag, so filtering on 'build' or 'test' can happen
processBuilder.Tags = ReadOnlyArray <StringId> .FromWithoutCopy(new[] { StringId.Create(m_context.StringTable, project.ScriptCommandName) });
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
var logDirectory = GetLogDirectory(project);
processBuilder.SetStandardOutputFile(logDirectory.Combine(m_context.PathTable, "build.log"));
processBuilder.SetStandardErrorFile(logDirectory.Combine(m_context.PathTable, "error.log"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, " "))
{
// Execute the command and redirect the output to a designated log file
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.ScriptCommand));
// If we need to append arguments to the script command, do it here
if (m_customCommands.TryGetValue(project.ScriptCommandName, out IReadOnlyList <JavaScriptArgument> extraArguments))
{
foreach (JavaScriptArgument value in extraArguments)
{
AddJavaScriptArgumentToBuilder(processBuilder.ArgumentsBuilder, value);
}
}
}
}
FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable);
}
private FullSymbol GetFullSymbolFromProject(RushProject project)
{
// We construct the name of the value using the project name
var valueName = PipConstructionUtilities.SanitizeStringForSymbol(project.Name);
var fullSymbol = FullSymbol.Create(m_context.SymbolTable, valueName);
return(fullSymbol);
}
/// <summary>
/// Used also for tests to discover processes based on project names
/// </summary>
internal static FullSymbol GetFullSymbolFromProject(string projectName, string scriptCommandName, SymbolTable symbolTable)
{
// We construct the name of the value using the project name
var valueName = PipConstructionUtilities.SanitizeStringForSymbol($"{projectName}_{scriptCommandName}");
var fullSymbol = FullSymbol.Create(symbolTable, valueName);
return(fullSymbol);
}
private void SetEnvironmentVariables(ProcessBuilder processBuilder, NinjaNode node)
{
foreach (KeyValuePair <string, string> kvp in m_environmentVariables.Value)
{
if (kvp.Value != null)
{
var envPipData = new PipDataBuilder(m_context.StringTable);
if (SpecialEnvironmentVariables.PassThroughPrefixes.All(prefix => !kvp.Key.StartsWith(prefix)))
{
envPipData.Add(kvp.Value);
processBuilder.SetEnvironmentVariable(StringId.Create(m_context.StringTable, kvp.Key), envPipData.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
}
}
}
foreach (var kvp in m_passThroughEnvironmentVariables.Value)
{
processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, kvp.Key));
}
foreach (var envVar in SpecialEnvironmentVariables.CloudBuildEnvironment)
{
processBuilder.SetPassthroughEnvironmentVariable(StringId.Create(m_context.StringTable, envVar));
}
// GlobalUnsafePassthroughEnvironmentVariables
processBuilder.SetGlobalPassthroughEnvironmentVariable(m_frontEndHost.Configuration.FrontEnd.GlobalUnsafePassthroughEnvironmentVariables, m_context.StringTable);
// We will specify a different MSPDBSRV endpoint for every pip.
// This means every pip that needs to communicate to MSPDBSRV will
// spawn a different child process.
// This is because if two pips use the same endpoint at the same time
// then second one will fail after the first one finishes, because the
// first one was the one that spawned MSPDBSRV.EXE as a child
// (which gets killed).
//
// IMPORTANT: This will cause the build to fail if two pips target the same PDB file.
// Both LINK.EXE and CL.EXE can use MSPDBSRV.EXE:
// - If all linkers specify a different /pdb argument (or don't specify this argument and
// are linking programs with different names
// [https://docs.microsoft.com/en-us/cpp/build/reference/debug-generate-debug-info])
// then this won't happen
//
// - We're forcing the compiler to not output to PDBs (/Z7)
//
// so this should work in the normal cases.
var mspdbsrvPipDataBuilder = new PipDataBuilder(m_context.StringTable);
mspdbsrvPipDataBuilder.Add(PipConstructionUtilities.ComputeSha256(node.Command)); // Unique value for each pip
processBuilder.SetEnvironmentVariable(
StringId.Create(m_context.StringTable, SpecialEnvironmentVariables.MsPdvSrvEndpoint),
mspdbsrvPipDataBuilder.ToPipData(string.Empty, PipDataFragmentEscaping.NoEscaping));
}
public void RedirectedUserProfileSanitizesScriptCommandName(string scriptCommandName)
{
var project = CreateRushProject(scriptCommandName: scriptCommandName);
var processOutputDirectories = Start()
.Add(project)
.ScheduleAll()
.RetrieveSuccessfulProcess(project)
.DirectoryOutputs;
string sanitizedPath = PipConstructionUtilities.SanitizeStringForSymbol(scriptCommandName);
XAssert.IsTrue(processOutputDirectories.Any(outputDirectory => outputDirectory.Path.ToString(PathTable).Contains(sanitizedPath)));
}
private AbsolutePath GetLogDirectory(JavaScriptProject projectFile)
{
var success = Root.TryGetRelative(PathTable, projectFile.ProjectFolder, out var inFolderPathFromEnlistmentRoot);
Contract.Assert(success, $"Configuration root '{Root.ToString(PathTable)}' should be a parent of '{projectFile.ProjectFolder.ToString(PathTable)}'");
// We hardcode the log to go under the output directory Logs/<frontend-name> (and follow the project structure underneath)
var result = LogDirectoryBase(m_frontEndHost.Configuration, m_context.PathTable, m_resolverSettings.Name)
.Combine(PathTable, inFolderPathFromEnlistmentRoot)
.Combine(PathTable, PipConstructionUtilities.SanitizeStringForSymbol(projectFile.Name))
.Combine(PathTable, PipConstructionUtilities.SanitizeStringForSymbol(projectFile.ScriptCommandName));
return(result);
}
private AbsolutePath GetLogDirectory(RushProject projectFile)
{
var success = Root.TryGetRelative(PathTable, projectFile.ProjectFolder, out var inFolderPathFromEnlistmentRoot);
Contract.Assert(success, $"Configuration root '{Root.ToString(PathTable)}' should be a parent of '{projectFile.ProjectFolder.ToString(PathTable)}'");
// We hardcode the log to go under the output directory Logs/Rush (and follow the project structure underneath)
// The 'official' log directory (defined by Configuration.Logging) is not stable in CloudBuild across machines, and therefore it would
// introduce cache misses
var result = LogDirectoryBase(m_frontEndHost.Configuration, m_context.PathTable)
.Combine(PathTable, inFolderPathFromEnlistmentRoot)
.Combine(PathTable, PipConstructionUtilities.SanitizeStringForSymbol(projectFile.Name))
.Combine(PathTable, PipConstructionUtilities.SanitizeStringForSymbol(projectFile.ScriptCommandName));
return(result);
}
/// <summary>
/// Computes a sha256 based on the project properties
/// </summary>
private string ComputeSha256(ProjectWithPredictions projectWithPredictions)
{
using (var builderWrapper = Pools.GetStringBuilder())
{
// full path, global properties and targets to execute should uniquely identify the project within the build graph
StringBuilder builder = builderWrapper.Instance;
builder.Append(projectWithPredictions.FullPath.ToString(PathTable));
builder.Append("|");
builder.Append(projectWithPredictions.PredictedTargetsToExecute.IsDefaultTargetsAppended);
builder.Append(string.Join("|", projectWithPredictions.PredictedTargetsToExecute.Targets));
builder.Append("|");
builder.Append(string.Join("|", projectWithPredictions.GlobalProperties.Select(kvp => kvp.Key + "|" + kvp.Value)));
return(PipConstructionUtilities.ComputeSha256(builder.ToString()));
}
}
private void SetUntrackedFilesAndDirectories(ProcessBuilder processBuilder)
{
// On some machines, the current user and public user desktop.ini are read by Powershell.exe.
// Ignore accesses to the user profile and Public common user profile.
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile)));
if (Engine.TryGetBuildParameter("PUBLIC", m_frontEndName, out string publicDir))
{
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(AbsolutePath.Create(PathTable, publicDir)));
}
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
// Git accesses should be ignored if .git directory is there
var gitDirectory = Root.Combine(PathTable, ".git");
if (Engine.DirectoryExists(gitDirectory))
{
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(gitDirectory));
processBuilder.AddUntrackedFile(FileArtifact.CreateSourceFile(Root.Combine(PathTable, ".gitattributes")));
processBuilder.AddUntrackedFile(FileArtifact.CreateSourceFile(Root.Combine(PathTable, ".gitignore")));
}
}
private void UntrackFilesAndDirectories(ProcessBuilder processBuilder)
{
processBuilder.AddUntrackedWindowsDirectories();
processBuilder.AddUntrackedProgramDataDirectories();
processBuilder.AddUntrackedAppDataDirectories();
if (m_untrackingSettings != null)
{
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_untrackingSettings);
}
var programFilesDirectoryArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(AbsolutePath.Create(m_context.PathTable, Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles)));
var programFilesX86DirectoryArtifact = DirectoryArtifact.CreateWithZeroPartialSealId(AbsolutePath.Create(m_context.PathTable, Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86)));
processBuilder.AddUntrackedDirectoryScope(programFilesDirectoryArtifact);
processBuilder.AddUntrackedDirectoryScope(programFilesX86DirectoryArtifact);
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(AbsolutePath.Create(m_context.PathTable, @"C:\PROGRA~1\"))); // TODO: This but better
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(AbsolutePath.Create(m_context.PathTable, @"C:\PROGRA~2\"))); // TODO: This but better
// TODO: This is just here because the cloud build requires manually dropping the necessary executables and libraries, and should be removed
// when that issue is resolved.
string toolsDir = m_manuallyDroppedDependenciesPath.Value.ToString(m_context.PathTable);
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(AbsolutePath.Create(m_context.PathTable, toolsDir)));
// Git accesses should be ignored if .git directory is there
var gitDirectory = m_projectRoot.Combine(m_context.PathTable, ".git");
if (m_frontEndHost.Engine.DirectoryExists(gitDirectory))
{
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(gitDirectory));
processBuilder.AddUntrackedFile(FileArtifact.CreateSourceFile(m_projectRoot.Combine(m_context.PathTable, ".gitattributes")));
processBuilder.AddUntrackedFile(FileArtifact.CreateSourceFile(m_projectRoot.Combine(m_context.PathTable, ".gitignore")));
}
}
private PipConstructionHelper GetPipConstructionHelperForModule(AbsolutePath projectRoot, ModuleDefinition moduleDefinition, QualifierId qualifierId)
{
// One and only one AbsolutePath in the specs (corresponding to the build.ninja
// If this changed, this method would be out of here, as we would need a different PipConstructionHelper
// for each spec
Contract.Requires(moduleDefinition.Specs.Count == 1);
// Get a symbol that is unique for this particular instance
var fullSymbol = FullSymbol.Create(m_context.SymbolTable, $"ninja.{PipConstructionUtilities.SanitizeStringForSymbol(moduleDefinition.Descriptor.Name)}"); // TODO: Figure this out, complete
AbsolutePath pathToSpec = moduleDefinition.Specs.First();
if (!projectRoot.TryGetRelative(m_context.PathTable, pathToSpec, out var specRelativePath))
{
// Issue a warning and continue with Invalid path. PipConstructionHelper will just ignore
Tracing.Logger.Log.CouldNotComputeRelativePathToSpec(m_context.LoggingContext,
Location.FromFile(projectRoot.ToString(m_context.PathTable)),
projectRoot.ToString(m_context.PathTable),
pathToSpec.ToString(m_context.PathTable));
specRelativePath = RelativePath.Invalid;
}
var pipConstructionHelper = PipConstructionHelper.Create(
m_context,
m_frontEndHost.Engine.Layout.ObjectDirectory,
m_frontEndHost.Engine.Layout.RedirectedDirectory,
m_frontEndHost.Engine.Layout.TempDirectory,
m_frontEndHost.PipGraph,
moduleDefinition.Descriptor.Id,
moduleDefinition.Descriptor.Name,
specRelativePath,
fullSymbol,
new LocationData(pathToSpec, 0, 0), // TODO: This is the location of the value (that is scheduling pips through this helper) in its corresponding spec.
// Since we are not exposing any public value yet that represents this symbol, this location is fine.
qualifierId);
return(pipConstructionHelper);
}
/// <summary> /// Project-specific user profile folder /// </summary> internal static AbsolutePath UserProfile(JavaScriptProject project, PathTable pathTable) => project.TempFolder .Combine(pathTable, "USERPROFILE") .Combine(pathTable, PipConstructionUtilities.SanitizeStringForSymbol(project.ScriptCommandName));
