/// <summary>
/// The manifest is configured so all file accesses are allowed but reported, including child processes.
/// </summary>
/// <remarks>
/// Some special folders (Windows, InternetCache and History) are added as known scopes. Everything else will be flagged
/// as an 'unexpected' access. However, unexpected accesses are configured so they are not blocked.
/// </remarks>
private static FileAccessManifest CreateManifestToAllowAllAccesses(PathTable pathTable)
{
var fileAccessManifest = new FileAccessManifest(pathTable)
{
FailUnexpectedFileAccesses = false,
ReportFileAccesses = true,
MonitorChildProcesses = true,
};
fileAccessManifest.AddScope(
AbsolutePath.Create(pathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.Windows)),
FileAccessPolicy.MaskAll,
FileAccessPolicy.AllowAll);
fileAccessManifest.AddScope(
AbsolutePath.Create(pathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.InternetCache)),
FileAccessPolicy.MaskAll,
FileAccessPolicy.AllowAll);
fileAccessManifest.AddScope(
AbsolutePath.Create(pathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.History)),
FileAccessPolicy.MaskAll,
FileAccessPolicy.AllowAll);
return(fileAccessManifest);
}
private JsonSerializer ConstructProjectGraphSerializer()
{
var serializer = JsonSerializer.Create(ProjectGraphSerializationSettings.Settings);
// If the user profile has been redirected, we need to catch any path reported by MSBuild that falls under it
// and relocate it to the redirected user profile.
// This allows for cache hits across machines where the user profile is not uniformly located, and MSBuild
// happens to read a spec under it (the typical case is a props/target file under the nuget cache)
// Observe that the env variable UserProfile is already redirected in this case, and the engine abstraction exposes it.
// However, MSBuild very often manages to find the user profile by some other means
AbsolutePathJsonConverter absolutePathConverter;
if (m_configuration.Layout.RedirectedUserProfileJunctionRoot.IsValid)
{
// Let's get the redirected and original user profile folder
string redirectedUserProfile = SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile);
string originalUserProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
absolutePathConverter = new AbsolutePathJsonConverter(
m_context.PathTable,
AbsolutePath.Create(m_context.PathTable, originalUserProfile),
AbsolutePath.Create(m_context.PathTable, redirectedUserProfile)
);
}
else
{
absolutePathConverter = new AbsolutePathJsonConverter(m_context.PathTable);
}
serializer.Converters.Add(absolutePathConverter);
// Let's not add invalid absolute paths to any collection
serializer.Converters.Add(ValidAbsolutePathEnumerationJsonConverter.Instance);
return(serializer);
}
private FileAccessManifest GenerateFileAccessManifest(AbsolutePath toolDirectory, AbsolutePath outputFile)
{
// We make no attempt at understanding what the graph generation process is going to do
// We just configure the manifest to not fail on unexpected accesses, so they can be collected
// later if needed
var fileAccessManifest = new FileAccessManifest(m_context.PathTable)
{
FailUnexpectedFileAccesses = false,
ReportFileAccesses = true,
MonitorNtCreateFile = true,
MonitorZwCreateOpenQueryFile = true,
MonitorChildProcesses = true,
};
fileAccessManifest.AddScope(
AbsolutePath.Create(m_context.PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.Windows)),
FileAccessPolicy.MaskAll,
FileAccessPolicy.AllowAllButSymlinkCreation);
fileAccessManifest.AddScope(
AbsolutePath.Create(m_context.PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.InternetCache)),
FileAccessPolicy.MaskAll,
FileAccessPolicy.AllowAllButSymlinkCreation);
fileAccessManifest.AddScope(
AbsolutePath.Create(m_context.PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.History)),
FileAccessPolicy.MaskAll,
FileAccessPolicy.AllowAllButSymlinkCreation);
fileAccessManifest.AddScope(toolDirectory, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowReadAlways);
fileAccessManifest.AddPath(outputFile, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowWrite);
return(fileAccessManifest);
}
public void PathOption()
{
CommandLineUtilities.Option opt = default(CommandLineUtilities.Option);
opt.Name = "Switch";
opt.Value = "test.dll";
Assert.Equal(Path.Combine(Directory.GetCurrentDirectory(), "test.dll"), CommandLineUtilities.ParsePathOption(opt));
// Paths may be wrapped in quotes
opt.Value = "\"File With Spaces.dll\"";
Assert.Equal(Path.Combine(Directory.GetCurrentDirectory(), "File With Spaces.dll"), CommandLineUtilities.ParsePathOption(opt));
if (!OperatingSystemHelper.IsUnixOS)
{
opt.Name = "Switch";
opt.Value = SpecialFolderUtilities.GetFolderPath(SpecialFolder.Windows);
Assert.Equal(opt.Value, CommandLineUtilities.ParsePathOption(opt));
}
Assert.Throws <InvalidArgumentException>(() =>
{
opt.Name = "Switch";
opt.Value = null;
CommandLineUtilities.ParsePathOption(opt);
});
Assert.Throws <InvalidArgumentException>(() =>
{
opt.Name = "Switch";
opt.Value = string.Empty;
CommandLineUtilities.ParsePathOption(opt);
});
}
/// <summary>
/// Constructor
/// </summary>
public PipEnvironment(LoggingContext loggingContext)
{
m_loggingContext = loggingContext;
FullEnvironmentVariables =
GetFactory(ReportDuplicateVariable)
.PopulateFromEnvironment();
#if PLATFORM_WIN
var comspec = Path.Combine(SpecialFolderUtilities.SystemDirectory, "cmd.exe");
var pathExt = ".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC";
#endif
var path =
string.Join(
OperatingSystemHelper.IsUnixOS ? ":" : ";",
SpecialFolderUtilities.SystemDirectory,
#if PLATFORM_WIN
SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.Windows),
Path.Combine(SpecialFolderUtilities.SystemDirectory, "wbem")
#else
MacPaths.UsrBin,
MacPaths.UsrSbin,
MacPaths.Bin,
MacPaths.Sbin
#endif
);
// the environment variable names below should use the casing appropriate for the target OS
// (on Windows it won't matter, but on Unix-like systems, including Cygwin environment on Windows,
// it matters, and has to be all upper-cased). See also doc comment for IBuildParameters.Select
m_baseEnvironmentVariables = FullEnvironmentVariables
.Select(new[]
{
"NUMBER_OF_PROCESSORS",
"OS",
"PROCESSOR_ARCHITECTURE",
"PROCESSOR_IDENTIFIER",
"PROCESSOR_LEVEL",
"PROCESSOR_REVISION",
"SystemDrive",
"SystemRoot",
"SYSTEMTYPE",
})
.Override(new Dictionary <string, string>()
{
{ "PATH", path },
#if PLATFORM_WIN
{ "ComSpec", comspec },
{ "PATHEXT", pathExt }
#endif
})
.Override(DisallowedTempVariables
.Select(tmp => new KeyValuePair <string, string>(tmp, RestrictedTemp)));
}
private static DirectoryArtifact GetSpecialFolder(PathTable pathTable, Environment.SpecialFolder specialFolder, params string[] subFolders)
{
var root = AbsolutePath.Create(pathTable, SpecialFolderUtilities.GetFolderPath(specialFolder));
if (subFolders != null)
{
foreach (var subFolder in subFolders)
{
root = root.Combine(pathTable, subFolder);
}
}
return(DirectoryArtifact.CreateWithZeroPartialSealId(root));
}
/// <summary>
/// Adds a system mount that is statically defined at the start of the build.
/// </summary>
private void AddStaticSystemMount(string name, Environment.SpecialFolder specialFolder, bool allowCreateDirectory = false, bool trackSourceFileChanges = false)
{
string folder = null;
try
{
// Don't verify the path for the sake of performance and also because if the path is verified and doesn't
// exist, an empty string will be returned. We want to unconditionally create the mount whether the backing
// path exists or not.
folder = SpecialFolderUtilities.GetFolderPath(specialFolder, Environment.SpecialFolderOption.DoNotVerify);
}
catch (ArgumentException)
{
Logger.Log.CouldNotAddSystemMount(m_loggingContext, name, folder);
return;
}
AddStaticSystemMount(name, folder, allowCreateDirectory, trackSourceFileChanges);
}
private static DirectoryArtifact GetSpecialFolder(PathTable pathTable, Environment.SpecialFolder specialFolder, params string[] subFolders)
{
// GetFolderPath will return empty paths for special folders that don't exist in the current user profile.
// Return DirectoryArtifact.Invalid for those folders so they can be omitted from being untracked when
// the system does not support them.
if (AbsolutePath.TryCreate(pathTable, SpecialFolderUtilities.GetFolderPath(specialFolder), out var root))
{
if (subFolders != null)
{
foreach (var subFolder in subFolders)
{
root = root.Combine(pathTable, subFolder);
}
}
return(DirectoryArtifact.CreateWithZeroPartialSealId(root));
}
return(DirectoryArtifact.Invalid);
}
public void SingletonPathOption()
{
CommandLineUtilities.Option opt = default(CommandLineUtilities.Option);
opt.Name = "Switch";
opt.Value = "test.dll";
Assert.Equal(Path.Combine(Directory.GetCurrentDirectory(), "test.dll"), CommandLineUtilities.ParseSingletonPathOption(opt, null));
if (!OperatingSystemHelper.IsUnixOS)
{
opt.Name = "Switch";
opt.Value = SpecialFolderUtilities.GetFolderPath(SpecialFolder.Windows);
Assert.Equal(opt.Value, CommandLineUtilities.ParseSingletonPathOption(opt, null));
}
Assert.Throws <InvalidArgumentException>(() =>
{
opt.Name = "Switch";
opt.Value = null;
CommandLineUtilities.ParseSingletonPathOption(opt, null);
});
Assert.Throws <InvalidArgumentException>(() =>
{
opt.Name = "Switch";
opt.Value = string.Empty;
CommandLineUtilities.ParseSingletonPathOption(opt, null);
});
Assert.Throws <InvalidArgumentException>(() =>
{
opt.Name = "Switch";
opt.Value = "New";
CommandLineUtilities.ParseSingletonPathOption(opt, "Existing");
});
}
private void SetUntrackedFilesAndDirectories(ProcessBuilder processBuilder)
{
// On some machines, the current user and public user desktop.ini are read by Powershell.exe.
// Ignore accesses to the user profile and Public common user profile.
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile)));
if (Engine.TryGetBuildParameter("PUBLIC", m_frontEndName, out string publicDir))
{
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(AbsolutePath.Create(PathTable, publicDir)));
}
PipConstructionUtilities.UntrackUserConfigurableArtifacts(processBuilder, m_resolverSettings);
// Git accesses should be ignored if .git directory is there
var gitDirectory = Root.Combine(PathTable, ".git");
if (Engine.DirectoryExists(gitDirectory))
{
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(gitDirectory));
processBuilder.AddUntrackedFile(FileArtifact.CreateSourceFile(Root.Combine(PathTable, ".gitattributes")));
processBuilder.AddUntrackedFile(FileArtifact.CreateSourceFile(Root.Combine(PathTable, ".gitignore")));
}
}
/// <summary>
/// Configures the process builder to execute the specified commands
/// </summary>
protected virtual void ConfigureProcessBuilder(
ProcessBuilder processBuilder,
JavaScriptProject project)
{
SetCmdTool(processBuilder, project);
// Working directory - the directory where the project file lives.
processBuilder.WorkingDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(project.ProjectFolder);
// We allow undeclared inputs to be read
processBuilder.Options |= Process.Options.AllowUndeclaredSourceReads;
// We want to enforce the use of weak fingerprint augmentation since input predictions could be not complete/sufficient
// to avoid a large number of path sets
processBuilder.Options |= Process.Options.EnforceWeakFingerprintAugmentation;
// Try to preserve path set casing since many JavaScript projects deal with paths in a case-sensitive way
// Otherwise in Windows we force path sets to be all uppercase
processBuilder.Options |= Process.Options.PreservePathSetCasing;
// By default the double write policy is to allow same content double writes and safe rewrites.
processBuilder.RewritePolicy |= RewritePolicy.DefaultSafe;
// Untrack the user profile. The corresponding mount is already configured for not tracking source files, and with allowed undeclared source reads,
// any attempt to read into the user profile will fail to compute its corresponding hash
processBuilder.AddUntrackedDirectoryScope(DirectoryArtifact.CreateWithZeroPartialSealId(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile)));
// Add the associated build script name as a tag, so filtering on 'build' or 'test' can happen
processBuilder.Tags = ReadOnlyArray <StringId> .FromWithoutCopy(new[] { StringId.Create(m_context.StringTable, project.ScriptCommandName) });
// Configure the pip to fail if stderr is written. Defaults to false if not explicitly configured.
if (m_resolverSettings.WritingToStandardErrorFailsExecution == true)
{
processBuilder.Options |= Process.Options.WritingToStandardErrorFailsExecution;
}
PipConstructionUtilities.UntrackUserConfigurableArtifacts(m_context.PathTable, project.ProjectFolder, m_allProjectRoots, processBuilder, m_resolverSettings);
var logDirectory = GetLogDirectory(project);
processBuilder.SetStandardOutputFile(logDirectory.Combine(m_context.PathTable, "build.log"));
processBuilder.SetStandardErrorFile(logDirectory.Combine(m_context.PathTable, "error.log"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString("/C"));
using (processBuilder.ArgumentsBuilder.StartFragment(PipDataFragmentEscaping.NoEscaping, " "))
{
// Execute the command and redirect the output to a designated log file
processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromString(project.ScriptCommand));
// If we need to append arguments to the script command, do it here
if (m_customCommands.TryGetValue(project.ScriptCommandName, out IReadOnlyList <JavaScriptArgument> extraArguments))
{
foreach (JavaScriptArgument value in extraArguments)
{
AddJavaScriptArgumentToBuilder(processBuilder.ArgumentsBuilder, value);
}
}
}
}
FrontEndUtilities.SetProcessEnvironmentVariables(CreateEnvironment(project), m_userDefinedPassthroughVariables, processBuilder, m_context.PathTable);
}
public void TestRedirectUserProfileDirectory()
{
// first run to create all necessary directories leading to obj directory
SetupTestData();
RunEngine();
string currentUserProfile = SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile);
string junctionPath = Path.Combine(Configuration.Layout.ObjectDirectory.ToString(Context.PathTable), "buildXLUserProfile");
bool specialFolderInitializerWasCalled = false;
var translatedDirectory = new List <TranslateDirectoryData>();
var properties = new Dictionary <string, string>();
var expectedProperties = new Dictionary <string, string>
{
{ "APPDATA", Path.Combine(junctionPath, "AppData", "Roaming") },
{ "LOCALAPPDATA", Path.Combine(junctionPath, "AppData", "Local") },
{ "USERPROFILE", junctionPath },
{ "USERNAME", "buildXLUserProfile" },
{ "HOMEDRIVE", Path.GetPathRoot(junctionPath).TrimEnd('\\') },
{ "HOMEPATH", junctionPath.Substring(Path.GetPathRoot(junctionPath).TrimEnd('\\').Length) },
{ "INTERNETCACHE", Path.Combine(junctionPath, "AppData", "Local", "Microsoft", "Windows", "INetCache") },
{ "INTERNETHISTORY", Path.Combine(junctionPath, "AppData", "Local", "Microsoft", "Windows", "History") },
{ "INETCOOKIES", Path.Combine(junctionPath, "AppData", "Local", "Microsoft", "Windows", "INetCookies") },
{ "LOCALLOW", Path.Combine(junctionPath, "AppData", "LocalLow") },
};
// add the variables to the dictionary, so we can verify that the method overrides the existing values
foreach (var envVar in expectedProperties.Keys)
{
properties.Add(envVar, string.Empty);
}
try
{
var success = BuildXLEngine.RedirectUserProfileDirectory(
Configuration.Layout.ObjectDirectory,
translatedDirectory,
properties,
dict => { specialFolderInitializerWasCalled = true; },
true,
Context.PathTable,
LoggingContext);
// must have finished successfully
XAssert.IsTrue(success);
// verify the env block is properly populated
XAssert.AreSetsEqual(expectedProperties.Keys, properties.Keys, true);
XAssert.IsFalse(expectedProperties.Any(kvp => properties[kvp.Key] != kvp.Value));
XAssert.IsTrue(specialFolderInitializerWasCalled);
// verify junction
var openResult = FileUtilities.TryOpenDirectory(junctionPath, FileDesiredAccess.FileReadAttributes, FileShare.ReadWrite, FileFlagsAndAttributes.FileFlagOpenReparsePoint, out var handle);
XAssert.IsTrue(openResult.Succeeded);
using (handle)
{
var possibleTarget = FileUtilities.TryGetReparsePointTarget(handle, junctionPath);
XAssert.IsTrue(possibleTarget.Succeeded);
XAssert.AreEqual(FileSystemWin.NtPathPrefix + currentUserProfile, possibleTarget.Result);
}
// verify that we added a new directory translation
AbsolutePath.TryCreate(Context.PathTable, currentUserProfile, out var fromPath);
AbsolutePath.TryCreate(Context.PathTable, junctionPath, out var toPath);
XAssert.IsTrue(translatedDirectory.Count == 1);
XAssert.IsTrue(translatedDirectory[0].FromPath == fromPath && translatedDirectory[0].ToPath == toPath);
}
finally
{
// clean the junction after the test
var possibleProbe = FileUtilities.TryProbePathExistence(junctionPath, false);
if (possibleProbe.Succeeded && possibleProbe.Result != PathExistence.Nonexistent)
{
// we attempt to delete the junction, but we do not care if we failed to do
FileUtilities.TryRemoveDirectory(junctionPath, out int errCode);
}
}
}
public void RedirectedUserProfileIsHonored()
{
// Create a project directly under the user profile
var pathToTestProj = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "test.csproj");
try
{
// We still need an entry point under the repo root, so we create a 'dirs' project for that
var config = (CommandLineConfiguration)Build()
.AddSpec("dirs.proj", CreateDirsProject(pathToTestProj))
.AddSpec(pathToTestProj, CreateHelloWorldProject())
.PersistSpecsAndGetConfiguration();
// Set a redirected profile root to point to the test root
var redirectedProfile = AbsolutePath.Create(PathTable, TestRoot);
config.Layout.RedirectedUserProfileJunctionRoot = redirectedProfile;
var engineResult = RunEngineWithConfig(config);
Assert.True(engineResult.IsSuccess);
// The test project file for the corresponding pip should be located in the redirected user profile
var testProj = (Process)engineResult.EngineState.PipGraph
.RetrievePipsOfType(PipType.Process)
.Single(p => RetrieveProcessArguments((Process)p).Contains(SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile)));
// There shouldn't be a statically declared input for test.csproj under the redirected profile since the user profile has a corresponding mount
// with hash source file disabled, so static declarations under it are skipped. But so the declaration was properly redirected
Assert.False(testProj.Dependencies.Any(input =>
input.Path.IsWithin(PathTable, redirectedProfile) &&
input.Path.GetName(PathTable) == PathAtom.Create(StringTable, "test.csproj"))
);
}
finally
{
File.Delete(pathToTestProj);
}
}
