public Process Build()
{
Contract.Assert(m_executable != null, "WithExecutable method should be called before calling Build() method");
return(new Process(
executable: m_executable,
workingDirectory: m_workingDirectory,
arguments: m_arguments ?? m_argumentsFactory(this),
responseFile: FileArtifact.Invalid,
responseFileData: PipData.Invalid,
environmentVariables: ReadOnlyArray <EnvironmentVariable> .From(m_environmentVariables ?? Enumerable.Empty <EnvironmentVariable>()),
standardInput: FileArtifact.Invalid,
standardOutput: FileArtifact.Invalid,
standardError: FileArtifact.Invalid,
standardDirectory: m_standardDirectory,
warningTimeout: null,
timeout: null,
dependencies: ReadOnlyArray <FileArtifact> .From(CreateDependencies()),
outputs: ReadOnlyArray <FileArtifactWithAttributes> .From(Outputs),
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .From(m_directoryDependencies ?? Enumerable.Empty <DirectoryArtifact>()),
directoryOutputs: ReadOnlyArray <DirectoryArtifact> .Empty,
orderDependencies: ReadOnlyArray <PipId> .From(m_orderDependencies ?? Enumerable.Empty <PipId>()),
untrackedPaths: ReadOnlyArray <AbsolutePath> .From(m_untrackedPathes ?? Enumerable.Empty <AbsolutePath>()),
untrackedScopes: ReadOnlyArray <AbsolutePath> .From(m_untrackedScopes ?? Enumerable.Empty <AbsolutePath>()),
tags: ReadOnlyArray <StringId> .From(m_tags ?? Enumerable.Empty <StringId>()),
successExitCodes: ReadOnlyArray <int> .Empty,
semaphores: ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: m_pipProvenance ?? PipProvenance.CreateDummy(m_context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty,
options: m_options,
serviceInfo: m_serviceInfo,
preserveOutputWhitelist: ReadOnlyArray <AbsolutePath> .From(m_preserveOutputWhitelist ?? Enumerable.Empty <AbsolutePath>()),
changeAffectedInputListWrittenFile: m_changeAffectedInputListWrittenFile));
}
/// <nodoc />
public MacOsDefaults(PathTable pathTable, PipGraph.Builder pipGraph)
{
m_provenance = new PipProvenance(
0,
ModuleId.Invalid,
StringId.Invalid,
FullSymbol.Invalid,
LocationData.Invalid,
QualifierId.Unqualified,
PipData.Invalid);
m_inputFiles =
new[]
{
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.Etc)),
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.TmpDir)),
};
// Sealed Source inputs
m_inputDirectories =
new[]
{
GetSourceSeal(pathTable, pipGraph, MacPaths.Applications),
GetSourceSeal(pathTable, pipGraph, MacPaths.UsrBin),
GetSourceSeal(pathTable, pipGraph, MacPaths.UsrInclude),
GetSourceSeal(pathTable, pipGraph, MacPaths.UsrLib),
GetSourceSeal(pathTable, pipGraph, MacPaths.Library),
GetSourceSeal(pathTable, pipGraph, MacPaths.UserProvisioning),
};
m_untrackedFiles =
new[]
{
// login.keychain is created by the OS the first time any process invokes an OS API that references the keychain.
// Untracked because build state will not be stored there and code signing will fail if required certs are in the keychain
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserKeyChainsDb)),
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserKeyChains)),
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserCFTextEncoding)),
};
m_untrackedDirectories =
new[]
{
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Bin),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Dev),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Private),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Sbin),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.SystemLibrary),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrLibexec),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrShare),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrStandalone),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrSbin),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Var),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UserPreferences),
};
}
private static ObjectInfo ProvenanceInfo(PipProvenance prov)
{
return(new ObjectInfo(
prov.ToString(),
new[]
{
new Property("Qualifier", prov.QualifierId),
new Property("Output", prov.OutputValueSymbol),
new Property("Usage", prov.Usage),
}));
}
private static Pip CreateCmdPip(BuildXLContext context, string tempDirectory, string outFile, bool is64Bit)
{
Contract.Requires(context != null);
Contract.Requires(tempDirectory != null);
Contract.Requires(!string.IsNullOrEmpty(outFile));
var pathTable = context.PathTable;
string executable = is64Bit ? CmdHelper.CmdX64 : CmdHelper.CmdX86;
FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
string workingDirectory = AssemblyDirectory;
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
AbsolutePath outFilePath = AbsolutePath.Create(pathTable, outFile);
FileArtifact outFileArtifact = FileArtifact.CreateSourceFile(outFilePath).CreateNextWrittenVersion();
var pip = new BuildXL.Pips.Operations.Process(
executableArtifact,
workingDirectoryAbsolutePath,
PipDataBuilder.CreatePipData(
context.StringTable,
" ",
PipDataFragmentEscaping.CRuntimeArgumentRules,
"/d",
"/c",
"echo",
"hello",
">",
outFileArtifact),
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .Empty,
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
AbsolutePath.Create(pathTable, tempDirectory),
null,
null,
ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact),
ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(outFileArtifact.WithAttributes()),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencies(pathTable)),
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(pathTable)),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
return(pip);
}
internal PipProvenance CreatePipProvenance(PipData usage)
{
var result = new PipProvenance(
GetNextSemiStableHash(),
moduleId: m_moduleId,
moduleName: StringId.Create(Context.StringTable, m_moduleName),
outputValueSymbol: m_valuePip.Symbol,
token: m_valuePip.LocationData,
qualifierId: m_valuePip.Qualifier,
usage: usage);
return(result);
}
/// <summary>
/// Adds a fake process pip that produces only the given path.
/// </summary>
public Process AddProcess(AbsolutePath producedPath, DoubleWritePolicy doubleWritePolicy = DoubleWritePolicy.DoubleWritesAreErrors)
{
Contract.Assume(!m_pathProducers.ContainsKey(producedPath), "Each path may have only one producer (no rewrites)");
AbsolutePath workingDirectory = AbsolutePath.Create(m_context.PathTable, PathGeneratorUtilities.GetAbsolutePath("X", ""));
AbsolutePath exe = AbsolutePath.Create(m_context.PathTable, PathGeneratorUtilities.GetAbsolutePath("X", "fake.exe"));
var process = new Process(
executable: FileArtifact.CreateSourceFile(exe),
workingDirectory: workingDirectory,
arguments: PipDataBuilder.CreatePipData(m_context.StringTable, string.Empty, PipDataFragmentEscaping.NoEscaping),
responseFile: FileArtifact.Invalid,
responseFileData: PipData.Invalid,
environmentVariables: ReadOnlyArray <EnvironmentVariable> .Empty,
standardInput: FileArtifact.Invalid,
standardOutput: FileArtifact.Invalid,
standardError: FileArtifact.Invalid,
standardDirectory: workingDirectory,
warningTimeout: null,
timeout: null,
dependencies: ReadOnlyArray <FileArtifact> .FromWithoutCopy(FileArtifact.CreateSourceFile(exe)),
outputs: ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(FileArtifact.CreateSourceFile(producedPath).CreateNextWrittenVersion().WithAttributes()),
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty,
directoryOutputs: ReadOnlyArray <DirectoryArtifact> .Empty,
orderDependencies: ReadOnlyArray <PipId> .Empty,
untrackedPaths: ReadOnlyArray <AbsolutePath> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .Empty,
tags: ReadOnlyArray <StringId> .Empty,
successExitCodes: ReadOnlyArray <int> .Empty,
semaphores: ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(m_context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty,
doubleWritePolicy: doubleWritePolicy);
process.PipId = AllocateNextPipId();
m_pips.Add(process.PipId, process);
m_pathProducers.Add(producedPath, process);
return(process);
}
private static Process CreateDummyProcess(PipExecutionContext context, PipId pipId)
{
var exe = FileArtifact.CreateSourceFile(AbsolutePath.Create(context.PathTable, X("/X/exe")));
List <FileArtifact> dependencies = new List <FileArtifact> {
exe
};
var p = new Process(
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty,
executable: exe,
workingDirectory: AbsolutePath.Create(context.PathTable, X("/X")),
arguments: new PipDataBuilder(context.StringTable).ToPipData(" ", PipDataFragmentEscaping.NoEscaping),
responseFile: FileArtifact.Invalid,
responseFileData: PipData.Invalid,
environmentVariables: ReadOnlyArray <EnvironmentVariable> .Empty,
standardInput: FileArtifact.Invalid,
standardOutput: FileArtifact.Invalid,
standardError: FileArtifact.Invalid,
standardDirectory: AbsolutePath.Create(context.PathTable, X("/X/std")),
warningTimeout: null,
timeout: null,
dependencies: ReadOnlyArray <FileArtifact> .From(dependencies),
outputs: ReadOnlyArray <FileArtifactWithAttributes> .Empty,
directoryOutputs: ReadOnlyArray <DirectoryArtifact> .Empty,
orderDependencies: ReadOnlyArray <PipId> .Empty,
untrackedPaths: ReadOnlyArray <AbsolutePath> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .Empty,
tags: ReadOnlyArray <StringId> .Empty,
successExitCodes: ReadOnlyArray <int> .Empty,
semaphores: ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty)
{
PipId = pipId
};
return(p);
}
/// <summary>
/// Creates a process pip.
/// </summary>
protected Process CreateProcess(
IEnumerable <Operation> processOperations,
IEnumerable <string> tags = null,
IEnumerable <EnvironmentVariable> environmentVariables = null,
IEnumerable <DirectoryArtifact> directoryDependencies = null,
IEnumerable <DirectoryArtifact> directoryOutputs = null,
IEnumerable <PipId> orderDependencies = null,
IEnumerable <AbsolutePath> untrackedPaths = null,
IEnumerable <AbsolutePath> untrackedScopes = null,
FileArtifact?stdOut = null,
FileArtifact?stdError = null,
PipProvenance provenance = null,
IEnumerable <AbsolutePath> additionalTempDirectories = null,
AbsolutePath?tempDirectory = null,
IEnumerable <FileArtifact> tempFiles = null /* converted to FileArtifactWithAttributes with FileExistence.Temporary, then concatenated to outputs */)
{
FileArtifact executable = TestProcessExecutable;
var dao = InferIOFromOperations(processOperations);
return(CreateProcessWithoutInferringAccesses(
dependencies: dao.Dependencies,
outputs: dao.Outputs,
processOperations: processOperations,
tags: tags,
environmentVariables: environmentVariables,
directoryDependencies: directoryDependencies,
directoryOutputs: directoryOutputs,
orderDependencies: orderDependencies,
untrackedPaths: untrackedPaths,
untrackedScopes: untrackedScopes,
stdOut: stdOut,
stdError: stdError,
provenance: provenance,
additionalTempDirectories: additionalTempDirectories,
tempDirectory: tempDirectory,
tempFiles: tempFiles));
}
/// <summary>
/// Generates the PipMetadata for a given Pip
/// </summary>
public PipMetadata GeneratePipMetadata(Pip pip)
{
PipMetadata pipMetadata = new PipMetadata
{
PipId = pip.PipId.Value,
SemiStableHash = pip.FormattedSemiStableHash,
PipType = pip.PipType,
Tags = pip.Tags.IsValid ? pip.Tags.Select(tag => tag.ToString(StringTable)).ToList() : null
};
pipMetadata.Tags = pipMetadata.Tags.Any() ? pipMetadata.Tags : null;
PipProvenance provenance = pip.Provenance;
pipMetadata.Qualifier = PipGraph.Context.QualifierTable.GetCanonicalDisplayString(provenance.QualifierId);
pipMetadata.Usage = provenance.Usage.IsValid ? provenance.Usage.ToString(PathTable) : null;
pipMetadata.SpecFilePath = provenance.Token.Path.ToString(PathTable);
pipMetadata.OutputValueSymbol = provenance.OutputValueSymbol.ToString(SymbolTable);
pipMetadata.ModuleId = provenance.ModuleId.Value;
pipMetadata.SpecFilePath = provenance.Token.Path.ToString(PathTable);
pipMetadata.PipDependencies = PipGraph.RetrievePipReferenceImmediateDependencies(pip.PipId, null)
.Where(pipRef => pipRef.PipType != PipType.HashSourceFile)
.Select(pipRef => pipRef.PipId)
.Select(pipId => PipTable.HydratePip(pipId, PipQueryContext.ViewerAnalyzer))
.Select(pipHash => pipHash.FormattedSemiStableHash)
.ToList();
pipMetadata.PipDependents = PipGraph.RetrievePipReferenceImmediateDependents(pip.PipId, null)
.Select(pipRef => pipRef.PipId)
.Select(pipId => PipTable.HydratePip(pipId, PipQueryContext.ViewerAnalyzer))
.Select(pipData => pipData.FormattedSemiStableHash)
.ToList();
pipMetadata.PipDependencies = pipMetadata.PipDependencies.Any() ? pipMetadata.PipDependencies : null;
pipMetadata.PipDependents = pipMetadata.PipDependents.Any() ? pipMetadata.PipDependents : null;
return(pipMetadata);
}
public static Process CreateDummyProcessWithInputs(IEnumerable <FileArtifact> inputs, BuildXLContext context)
{
FileArtifact output = FileArtifact.CreateSourceFile(AbsolutePath.Create(context.PathTable, @"\\FAKEPATH\output" + s_procCounter + ".txt")).CreateNextWrittenVersion();
s_procCounter++;
FileArtifact exe = FileArtifact.CreateSourceFile(AbsolutePath.Create(context.PathTable, @"\\FAKEPATH\tool.exe"));
var pipDataBuilder = new PipDataBuilder(context.StringTable);
return(new Process(
executable: exe,
workingDirectory: AbsolutePath.Create(context.PathTable, @"\\FAKEPATH\"),
arguments: pipDataBuilder.ToPipData(" ", PipDataFragmentEscaping.NoEscaping),
responseFile: FileArtifact.Invalid,
responseFileData: PipData.Invalid,
environmentVariables: ReadOnlyArray <EnvironmentVariable> .Empty,
standardInput: FileArtifact.Invalid,
standardOutput: FileArtifact.Invalid,
standardError: FileArtifact.Invalid,
standardDirectory: output.Path.GetParent(context.PathTable),
warningTimeout: null,
timeout: null,
dependencies: ReadOnlyArray <FileArtifact> .From(inputs.Union(new[] { exe })),
outputs: ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(output.WithAttributes()),
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty,
directoryOutputs: ReadOnlyArray <DirectoryArtifact> .Empty,
orderDependencies: ReadOnlyArray <PipId> .Empty,
untrackedPaths: ReadOnlyArray <AbsolutePath> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .Empty,
tags: ReadOnlyArray <StringId> .Empty,
successExitCodes: ReadOnlyArray <int> .Empty,
semaphores: ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty));
}
public ObjectInfo GetObjectInfo(object context, object obj)
{
obj = obj is EvaluationResult evalResult
? evalResult.Value
: obj;
if (obj == null || IsInvalid(obj))
{
return(s_nullObj);
}
if (obj.GetType().IsArray)
{
return(ArrayObjInfo(((IEnumerable)obj).Cast <object>().ToArray()));
}
var customResult = m_customRenderer?.Invoke(this, context, obj);
if (customResult != null)
{
return(customResult);
}
return(obj switch
{
ScopeLocals scope => new ObjectInfo(LocalsScopeName, null, Lazy.Create(() => GetLocalsForStackEntry(scope.EvalState, scope.FrameIndex))),
ScopePipGraph scope => PipGraphInfo(scope.Graph).WithPreview(PipGraphScopeName),
ScopeAllModules scope => ArrayObjInfo(scope.EvaluatedModules.ToArray()).WithPreview(EvaluatedModulesScopeName),
IModuleAndContext mc => GetObjectInfo(mc.Tree.RootContext, mc.Module),
ObjectInfo objInf => objInf,
IPipGraph graph => PipGraphInfo(graph),
Pip pip => GenericObjectInfo(pip, $"<{pip.PipType}>").Build(),
PipProvenance prov => ProvenanceInfo(prov),
EnvironmentVariable envVar => EnvironmentVariableInfo(envVar),
PipFragment pipFrag => PipFragmentInfo(context, pipFrag),
Thunk thunk => thunk.Value != null?GetObjectInfo(context, thunk.Value) : new ObjectInfo("<not evaluated>"),
FunctionLikeExpression lambda => LambdaInfo(lambda),
Closure cls => LambdaInfo(cls.Function),
SymbolAtom sym => new ObjectInfo(sym.ToString(StringTable)),
StringId id => new ObjectInfo(id.ToString(StringTable)),
PipId id => new ObjectInfo($"{id.Value}"),
UndefinedLiteral _ => new ObjectInfo("undefined", UndefinedLiteral.Instance),
UndefinedValue _ => new ObjectInfo("undefined", UndefinedValue.Instance),
AbsolutePath path => new ObjectInfo($"p`{path.ToString(PathTable)}`", path),
RelativePath path => new ObjectInfo($"r`{path.ToString(StringTable)}`", path),
PathAtom atom => new ObjectInfo($"a`{atom.ToString(StringTable)}`", atom),
FileArtifact file => new ObjectInfo($"f`{file.Path.ToString(PathTable)}`", file),
DirectoryArtifact dir => new ObjectInfo($"d`{dir.Path.ToString(PathTable)}`", dir),
int num => new ObjectInfo($"{num}"),
uint num => new ObjectInfo($"{num}"),
short num => new ObjectInfo($"{num}", (int)num),
long num => new ObjectInfo($"{num}"),
char ch => new ObjectInfo($"'{ch}'", ch.ToString()),
string str => new ObjectInfo($"\"{str}\"", str),
Enum e => new ObjectInfo($"{e.GetType().Name}.{e}", e),
NumberLiteral numLit => new ObjectInfo(numLit.UnboxedValue.ToString(), numLit),
Func <object> func => FuncObjInfo(func),
ArraySegment <object> arrSeg => ArrayObjInfo(arrSeg),
IEnumerable enu => new ObjectInfoBuilder().Preview("IEnumerable").Prop("Result", Lazy.Create <object>(() => enu.Cast <object>().ToArray())).Build(),
ArrayLiteral arrLit => ArrayObjInfo(arrLit.Values.Select(v => v.Value).ToArray()).WithOriginal(arrLit),
ModuleBinding binding => GetObjectInfo(context, binding.Body),
ErrorValue error => ErrorValueInfo(),
object o => GenericObjectInfo(o).Build(),
_ => s_nullObj
});
private static Process CreateConsoleProcessInContainer(
BuildXLContext context,
TempFileStorage tempFiles,
PathTable pt,
string arguments,
ReadOnlyArray <FileArtifactWithAttributes> outputFiles,
ReadOnlyArray <DirectoryArtifact> directoryOutputs,
ContainerIsolationLevel containerIsolationLevel = ContainerIsolationLevel.IsolateAllOutputs)
{
var executableFileArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(context.PathTable, CmdHelper.CmdX64));
var argumentBuilder = new PipDataBuilder(context.PathTable.StringTable);
argumentBuilder.Add("/d");
argumentBuilder.Add("/c");
using (argumentBuilder.StartFragment(PipDataFragmentEscaping.CRuntimeArgumentRules, " "))
{
foreach (var arg in arguments.Split(new[] { ' ' }, System.StringSplitOptions.RemoveEmptyEntries))
{
argumentBuilder.Add(arg);
}
}
string workingDirectory = tempFiles.GetUniqueDirectory();
var workingDirectoryAbsolutePath = AbsolutePath.Create(context.PathTable, workingDirectory);
string uniqueOutputDirectory = tempFiles.GetUniqueDirectory();
var uniqueOutputDirectoryPath = AbsolutePath.Create(context.PathTable, uniqueOutputDirectory);
string uniqueRedirectedOutputDirectory = tempFiles.GetUniqueDirectory("redirected");
var uniqueRedirectedOutputDirectoryPath = AbsolutePath.Create(context.PathTable, uniqueRedirectedOutputDirectory);
var pip = new Process(
executableFileArtifact,
workingDirectoryAbsolutePath,
argumentBuilder.ToPipData(" ", PipDataFragmentEscaping.NoEscaping),
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .FromWithoutCopy(),
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pt),
null,
null,
dependencies: ReadOnlyArray <FileArtifact> .FromWithoutCopy(new[] { executableFileArtifact }),
outputs: outputFiles,
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .Empty,
directoryOutputs: directoryOutputs,
orderDependencies: ReadOnlyArray <PipId> .Empty,
untrackedPaths: ReadOnlyArray <AbsolutePath> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .Empty,
tags: ReadOnlyArray <StringId> .Empty,
successExitCodes: ReadOnlyArray <int> .Empty,
semaphores: ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty,
options: Process.Options.NeedsToRunInContainer,
uniqueOutputDirectory: uniqueOutputDirectoryPath,
uniqueRedirectedDirectoryRoot: uniqueRedirectedOutputDirectoryPath,
containerIsolationLevel: containerIsolationLevel);
return(pip);
}
private static Pip CreateSelfPip(
BuildXLContext context,
string tempDirectory,
string outFile,
IEnumerable <string> restInstructions,
bool is64Bit)
{
Contract.Requires(context != null);
Contract.Requires(tempDirectory != null);
Contract.Requires(!string.IsNullOrEmpty(outFile));
var pathTable = context.PathTable;
string workingDirectory = AssemblyDirectory;
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
AbsolutePath appdataPath = AbsolutePath.Create(pathTable, Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData));
AbsolutePath windowsFolderPath = AbsolutePath.Create(pathTable, Environment.GetFolderPath(Environment.SpecialFolder.Windows));
string executable = is64Bit ? TestExecutableX64 : TestExecutableX86;
FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, Path.Combine(workingDirectory, executable)));
AbsolutePath outFilePath = AbsolutePath.Create(pathTable, outFile);
FileArtifact outFileArtifact = FileArtifact.CreateSourceFile(outFilePath).CreateNextWrittenVersion();
var pip = new BuildXL.Pips.Operations.Process(
executableArtifact,
workingDirectoryAbsolutePath,
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, restInstructions.Select(ri => (PipDataAtom)ri).ToArray()),
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .Empty,
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
AbsolutePath.Create(pathTable, tempDirectory),
null,
null,
ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact),
ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(outFileArtifact.WithAttributes()),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .Empty,
ReadOnlyArray <AbsolutePath> .FromWithoutCopy(
new[]
{
workingDirectoryAbsolutePath,
windowsFolderPath,
// For unknown reasons, on some machines the process is accessing LOCALAPPDATA\Microsoft\Windows\Temporary Internet Files\counters.dat
appdataPath,
}),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
return(pip);
}
public async Task Stress()
{
const int N = 5;
const int M = N * N;
var context = BuildXLContext.CreateInstanceForTesting();
var loggingContext = CreateLoggingContextForTest();
var pathTable = context.PathTable;
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var config = ConfigHelpers.CreateDefault(pathTable, tempFiles.GetUniqueFileName(), tempFiles);
using (var pipTable = new PipTable(
context.PathTable,
context.SymbolTable,
initialBufferSize: 1024,
maxDegreeOfParallelism: (Environment.ProcessorCount + 2) / 3,
debug: false))
{
var executionEnvironment = new PipQueueTestExecutionEnvironment(
context,
config,
pipTable,
Path.Combine(TestOutputDirectory, "temp"),
TryGetSubstSourceAndTarget(out string substSource, out string substTarget) ? (substSource, substTarget) : default((string, string)?),
GetSandboxConnection());
Func <RunnablePip, Task <PipResult> > taskFactory = async(runnablePip) =>
{
PipResult result;
var operationTracker = new OperationTracker(runnablePip.LoggingContext);
var pip = runnablePip.Pip;
using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, runnablePip.LoggingContext))
{
result = await TestPipExecutor.ExecuteAsync(operationContext, executionEnvironment, pip);
}
executionEnvironment.MarkExecuted(pip);
return(result);
};
string executable = CmdHelper.OsShellExe;
FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
// This is the only file artifact we reference without a producer. Rather than scheduling a hashing pip, let's just invent one (so fingerprinting can succeed).
executionEnvironment.AddWellKnownFile(executableArtifact, WellKnownContentHashes.UntrackedFile);
using (var phase1PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
{
// phase 1: create some files
var baseFileArtifacts = new List <FileArtifact>();
for (int i = 0; i < N; i++)
{
string destination = tempFiles.GetUniqueFileName();
AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
baseFileArtifacts.Add(destinationArtifact);
PipData contents = PipDataBuilder.CreatePipData(
context.StringTable,
" ",
PipDataFragmentEscaping.CRuntimeArgumentRules,
i.ToString(CultureInfo.InvariantCulture));
var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
var pipId = pipTable.Add((uint)(i + 1), writeFile);
var contentHash = ContentHashingUtilities.HashString(contents.ToString(pathTable));
executionEnvironment.AddExpectedWrite(writeFile, destinationArtifact, contentHash);
var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactory, 0);
runnable.Start(new OperationTracker(loggingContext), loggingContext);
runnable.SetDispatcherKind(DispatcherKind.IO);
phase1PipQueue.Enqueue(runnable);
}
phase1PipQueue.SetAsFinalized();
phase1PipQueue.DrainQueues();
await Task.WhenAll(
Enumerable.Range(0, 2).Select(
async range =>
{
using (var phase2PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
{
// phase 2: do some more with those files
var pips = new ConcurrentDictionary <PipId, Tuple <string, int> >();
var checkerTasks = new ConcurrentQueue <Task>();
Action <PipId, Task <PipResult> > callback =
(id, task) =>
{
XAssert.IsTrue(task.Status == TaskStatus.RanToCompletion);
XAssert.IsFalse(task.Result.Status.IndicatesFailure());
Tuple <string, int> t;
if (!pips.TryRemove(id, out t))
{
XAssert.Fail();
}
checkerTasks.Enqueue(
Task.Run(
() =>
{
string actual = File.ReadAllText(t.Item1).Trim();
// TODO: Make this async
XAssert.AreEqual(actual, t.Item2.ToString());
}));
};
var r = new Random(0);
for (int i = 0; i < M; i++)
{
int sourceIndex = r.Next(baseFileArtifacts.Count);
FileArtifact sourceArtifact = baseFileArtifacts[sourceIndex];
string destination = tempFiles.GetUniqueFileName();
AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
Pip pip;
DispatcherKind queueKind;
switch (r.Next(2))
{
case 0:
pip = new CopyFile(sourceArtifact, destinationArtifact, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
queueKind = DispatcherKind.IO;
executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
break;
case 1:
string workingDirectory =
OperatingSystemHelper.IsUnixOS ? "/tmp" :
Environment.GetFolderPath(Environment.SpecialFolder.Windows);
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
var pipData = OperatingSystemHelper.IsUnixOS ?
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "-c", "'", "cp", sourceArtifact, destinationArtifact, "'") :
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "/d", "/c", "copy", "/B", sourceArtifact, destinationArtifact);
queueKind = DispatcherKind.CPU;
pip = new Process(
executableArtifact,
workingDirectoryAbsolutePath,
pipData,
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .Empty,
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pathTable),
null,
null,
ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact, sourceArtifact),
ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(destinationArtifact.WithAttributes()),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencies(pathTable)),
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(pathTable)),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
break;
default:
Contract.Assert(false);
continue;
}
var pipId = pipTable.Add((uint)((range *M) + N + i + 1), pip);
Func <RunnablePip, Task> taskFactoryWithCallback = async(runnablePip) =>
{
var task = taskFactory(runnablePip);
var pipResult = await task;
callback(pipId, task);
};
var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactoryWithCallback, 0);
runnable.Start(new OperationTracker(loggingContext), loggingContext);
runnable.SetDispatcherKind(queueKind);
phase2PipQueue.Enqueue(runnable);
if (!pips.TryAdd(pipId, Tuple.Create(destination, sourceIndex)))
{
Contract.Assert(false);
}
}
phase2PipQueue.SetAsFinalized();
phase2PipQueue.DrainQueues();
XAssert.AreEqual(0, pips.Count);
await Task.WhenAll(checkerTasks);
}
}));
}
}
}
}
public void TestLogLimit()
{
var failingCopyFile1 = new CopyFile(FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)), CreateOutputFileArtifact(), ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(Context));
var failingCopyFile2 = new CopyFile(FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)), CreateOutputFileArtifact(), ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(Context));
PipGraphBuilder.AddCopyFile(failingCopyFile1);
PipGraphBuilder.AddCopyFile(failingCopyFile2);
Configuration.Logging.DumpFailedPipsLogLimit = 1;
var schedulerResult = RunScheduler().AssertFailure();
var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips");
SetExpectedFailures(2, 0);
AssertErrorCount();
AssertVerboseEventLogged(LogEventId.RuntimeDumpPipLiteLogLimitReached, count: 1);
Assert.True(Directory.GetFiles(logFolder).Length == 1);
}
public void TestPassingAndFailingPips()
{
var existantCopyFileSrc = CreateSourceFile();
var existantCopyFileDest = CreateOutputFileArtifact();
File.WriteAllText(existantCopyFileSrc.Path.ToString(Context.PathTable), "copy file test");
var failingCopyFile = new CopyFile(FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)), CreateOutputFileArtifact(), ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(Context));
var passingCopyFile = CreateCopyFile(existantCopyFileSrc, existantCopyFileDest);
PipGraphBuilder.AddCopyFile(failingCopyFile);
PipGraphBuilder.AddCopyFile(passingCopyFile);
var schedulerResult = RunScheduler().AssertFailure();
var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips");
var failingLogFile = Path.Combine(logFolder, $"{failingCopyFile.FormattedSemiStableHash}.json");
var passingLogFile = Path.Combine(logFolder, $"{passingCopyFile.FormattedSemiStableHash}.json");
SetExpectedFailures(1, 0); // One error logged for the failing pip
AssertErrorCount();
Assert.True(File.Exists(failingLogFile));
Assert.False(File.Exists(passingLogFile));
}
/// <summary>
/// Creates a process pip.
/// </summary>
protected Process CreateCmdProcess(
IEnumerable <FileArtifact> dependencies,
IEnumerable <FileArtifact> outputs,
PipData?arguments = null,
IEnumerable <string> tags = null,
PipProvenance provenance = null,
IEnumerable <DirectoryArtifact> directoryDependencies = null,
IEnumerable <PipId> orderDependencies = null,
IEnumerable <EnvironmentVariable> environmentVariables = null,
FileArtifact?stdOut = null,
FileArtifact?stdError = null,
bool withWarning = false,
string value = null,
IEnumerable <FileArtifact> untrackedFiles = null,
IEnumerable <DirectoryArtifact> directoryOutputs = null)
{
Contract.Requires(dependencies != null);
Contract.Requires(outputs != null);
FileArtifact executable = CmdExecutable;
IEnumerable <AbsolutePath> untrackedDependencies = CmdHelper.GetCmdDependencies(Context.PathTable);
if (untrackedFiles != null)
{
List <AbsolutePath> files = new List <AbsolutePath>(untrackedDependencies);
foreach (FileArtifact oneFile in untrackedFiles)
{
files.Add(oneFile.Path);
}
untrackedDependencies = files;
}
untrackedFiles = untrackedFiles ?? Enumerable.Empty <FileArtifact>();
var process =
new Process(
executable: executable,
workingDirectory: GetWorkingDirectory(),
arguments: arguments ?? CreateCmdArguments(Context.StringTable, dependencies.Concat(untrackedFiles), outputs, includeWarning: withWarning),
responseFile: FileArtifact.Invalid,
responseFileData: PipData.Invalid,
environmentVariables: environmentVariables != null
? ReadOnlyArray <EnvironmentVariable> .From(environmentVariables)
: ReadOnlyArray <EnvironmentVariable> .Empty,
standardInput: FileArtifact.Invalid,
standardOutput: stdOut ?? FileArtifact.Invalid,
standardError: stdError ?? FileArtifact.Invalid,
standardDirectory: GetStandardDirectory(),
warningTimeout: null,
timeout: null,
// TODO:1759: Fix response file handling. Should be able to appear in the dependencies list, but should appear in the graph as a WriteFile pip.
dependencies: ReadOnlyArray <FileArtifact> .From((new[] { executable /*, responseFile*/ }).Concat(dependencies ?? CollectionUtilities.EmptyArray <FileArtifact>())),
outputs: ReadOnlyArray <FileArtifactWithAttributes> .From(outputs.Select(o => o.WithAttributes()).ToArray()),
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .From(directoryDependencies ?? new DirectoryArtifact[0]),
directoryOutputs: ReadOnlyArray <DirectoryArtifact> .From(directoryOutputs ?? new DirectoryArtifact[0]),
orderDependencies: orderDependencies != null ? ReadOnlyArray <PipId> .From(orderDependencies) : ReadOnlyArray <PipId> .Empty,
untrackedPaths: ReadOnlyArray <AbsolutePath> .From(untrackedDependencies),
untrackedScopes: ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(Context.PathTable)),
tags: ConvertToStringIdArray(tags),
successExitCodes: ReadOnlyArray <int> .Empty,
semaphores: ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: provenance,
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty,
warningRegex:
withWarning
? new RegexDescriptor(StringId.Create(Context.StringTable, WarningRegexDescription), RegexOptions.IgnoreCase)
: default(RegexDescriptor));
return(process);
}
/// <nodoc />
public MacOsDefaults(PathTable pathTable, IMutablePipGraph pipGraph)
{
m_provenance = new PipProvenance(
0,
ModuleId.Invalid,
StringId.Invalid,
FullSymbol.Invalid,
LocationData.Invalid,
QualifierId.Unqualified,
PipData.Invalid);
m_sourceSealDirectoryPaths =
new[]
{
MacPaths.Applications,
MacPaths.Library,
MacPaths.UserProvisioning
}
.Select(p => AbsolutePath.Create(pathTable, p))
.ToArray();
// Sealed Source inputs
// (using Lazy so that these directories are sealed and added to the graph only if explicitly requested by a process)
m_lazySourceSealDirectories = Lazy.Create(() =>
new DefaultSourceSealDirectories(m_sourceSealDirectoryPaths.Select(p => GetSourceSeal(pipGraph, p)).ToArray()));
m_untrackedFiles =
new[]
{
// login.keychain is created by the OS the first time any process invokes an OS API that references the keychain.
// Untracked because build state will not be stored there and code signing will fail if required certs are in the keychain
MacPaths.Etc,
MacPaths.UserKeyChainsDb,
MacPaths.UserKeyChains,
MacPaths.UserCFTextEncoding,
MacPaths.TmpDir
}
.Select(p => FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, p)))
.ToArray();
m_untrackedDirectories =
new[]
{
MacPaths.Bin,
MacPaths.Dev,
MacPaths.Private,
MacPaths.Sbin,
MacPaths.SystemLibrary,
MacPaths.UsrBin,
MacPaths.UsrInclude,
MacPaths.UsrLibexec,
MacPaths.UsrShare,
MacPaths.UsrStandalone,
MacPaths.UsrSbin,
MacPaths.Var,
MacPaths.UserPreferences,
// it's important to untrack /usr/lib instead of creating a sealed source directory
// - the set of dynamically loaded libraries during an execution of a process is
// not necessarily deterministic, i.e., when the same process---which itself is
// deterministic---is executed multiple times on same inputs, the set of
// dynamically loaded libraries is not necessarily going to stay the same.
MacPaths.UsrLib
}
.Select(p => DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, p))
.ToArray();
}
public ProcessBuilder WithProvenance(PipProvenance pipProvenance)
{
m_pipProvenance = pipProvenance;
return(this);
}
/// <summary>
/// Creates a process pip without inferring any accesses from processOperations. This allows the consumer to specify
/// all file acceses
/// </summary>
protected Process CreateProcessWithoutInferringAccesses(
IEnumerable <FileArtifact> dependencies,
IEnumerable <FileArtifact> outputs,
IEnumerable <Operation> processOperations,
IEnumerable <string> tags = null,
IEnumerable <EnvironmentVariable> environmentVariables = null,
IEnumerable <DirectoryArtifact> directoryDependencies = null,
IEnumerable <DirectoryArtifact> directoryOutputs = null,
IEnumerable <PipId> orderDependencies = null,
IEnumerable <AbsolutePath> untrackedPaths = null,
IEnumerable <AbsolutePath> untrackedScopes = null,
FileArtifact?stdOut = null,
FileArtifact?stdError = null,
PipProvenance provenance = null,
IEnumerable <AbsolutePath> additionalTempDirectories = null,
AbsolutePath?tempDirectory = null,
IEnumerable <FileArtifact> tempFiles = null /* converted to FileArtifactWithAttributes with FileExistence.Temporary, then concatenated to outputs */)
{
Contract.Requires(dependencies != null);
Contract.Requires(outputs != null);
FileArtifact executable = TestProcessExecutable;
untrackedScopes = untrackedScopes ?? new AbsolutePath[0];
if (tempDirectory != null)
// Make temp directories untracked scopes to mimic pips built through PipBuilder
{
untrackedScopes = untrackedScopes.Concat(new AbsolutePath[] { (AbsolutePath)tempDirectory });
}
untrackedScopes = untrackedScopes.Concat(additionalTempDirectories ?? new AbsolutePath[0]);
// Add C://Windows dependency for test executable
untrackedScopes = untrackedScopes.Concat(TestProcessDependencies);
IEnumerable <FileArtifactWithAttributes> outputsWithAttributes = outputs.Select(o => o.WithAttributes());
if (tempFiles != null)
{
outputsWithAttributes = outputsWithAttributes.Concat(tempFiles.Select(tf => FileArtifactWithAttributes.FromFileArtifact(tf, FileExistence.Temporary)));
}
var process =
new Process(
executable: executable,
workingDirectory: GetWorkingDirectory(),
arguments: CreateArguments(processOperations, Context.StringTable),
responseFile: FileArtifact.Invalid,
responseFileData: PipData.Invalid,
environmentVariables: environmentVariables != null
? ReadOnlyArray <EnvironmentVariable> .From(environmentVariables)
: ReadOnlyArray <EnvironmentVariable> .Empty,
standardInput: FileArtifact.Invalid,
standardOutput: stdOut ?? FileArtifact.Invalid,
standardError: stdError ?? FileArtifact.Invalid,
standardDirectory: GetStandardDirectory(),
warningTimeout: null,
timeout: null,
dependencies: ReadOnlyArray <FileArtifact> .From((new[] { executable }).Concat(dependencies ?? CollectionUtilities.EmptyArray <FileArtifact>())),
outputs: ReadOnlyArray <FileArtifactWithAttributes> .From(outputsWithAttributes),
directoryDependencies: ReadOnlyArray <DirectoryArtifact> .From(directoryDependencies ?? new DirectoryArtifact[0]),
directoryOutputs: ReadOnlyArray <DirectoryArtifact> .From(directoryOutputs ?? new DirectoryArtifact[0]),
orderDependencies: orderDependencies != null ? ReadOnlyArray <PipId> .From(orderDependencies) : ReadOnlyArray <PipId> .Empty,
untrackedPaths: ReadOnlyArray <AbsolutePath> .From(untrackedPaths ?? ReadOnlyArray <AbsolutePath> .Empty),
untrackedScopes: ReadOnlyArray <AbsolutePath> .From(untrackedScopes),
tags: ConvertToStringIdArray(tags),
successExitCodes: ReadOnlyArray <int> .Empty,
semaphores: ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: provenance ?? CreateProvenance(),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .From(additionalTempDirectories ?? ReadOnlyArray <AbsolutePath> .Empty),
tempDirectory: tempDirectory ?? default(AbsolutePath)
);
return(process);
}
public void TestDumpPipLite()
{
var copyFile = new CopyFile(CreateSourceFile(), CreateOutputFileArtifact(), new List <StringId>().ToReadOnlyArray(), PipProvenance.CreateDummy(Context));
PipGraphBuilder.AddCopyFile(copyFile);
var schedulerResult = RunScheduler().AssertSuccess();
var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips");
var pipDumpFile = Path.Combine(logFolder, $"{copyFile.FormattedSemiStableHash}.json");
List <Option> options = new List <Option>
{
new Option()
{
Name = "o",
Value = schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable)
},
new Option()
{
Name = "p",
Value = copyFile.FormattedSemiStableHash
}
};
RunAnalyzer(schedulerResult, null, options);
Assert.True(Directory.Exists(logFolder));
Assert.True(File.Exists(pipDumpFile));
}
public void TestDumpPipLiteWithPassingAndFailingPips()
{
var passingCopyFile = new CopyFile(CreateSourceFile(), CreateOutputFileArtifact(), new List <StringId>().ToReadOnlyArray(), PipProvenance.CreateDummy(Context));
var failingCopyFile = new CopyFile(FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)), CreateOutputFileArtifact(), new List <StringId>().ToReadOnlyArray(), PipProvenance.CreateDummy(Context));
PipGraphBuilder.AddCopyFile(passingCopyFile);
PipGraphBuilder.AddCopyFile(failingCopyFile);
var schedulerResult = RunScheduler().AssertFailure();
var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips");
var failingPipDumpFile = Path.Combine(logFolder, $"{failingCopyFile.FormattedSemiStableHash}.json");
var passingPipDumpFile = Path.Combine(logFolder, $"{passingCopyFile.FormattedSemiStableHash}.json");
List <Option> options = new List <Option>
{
new Option()
{
Name = "d+"
}
};
RunAnalyzer(schedulerResult, null, options);
Assert.True(Directory.Exists(logFolder));
Assert.True(File.Exists(failingPipDumpFile));
Assert.True(!File.Exists(passingPipDumpFile));
}
private async Task ProcessWindowsCallHelper(
string functionName,
SandboxConfiguration config = null,
IEnumerable <string> extraDependencies = null,
IEnumerable <string> extraOutputs = null,
int callCount = 1,
string commandPrefix = "",
bool readsAndWritesDirectories = false,
bool untrackedOutputs = false,
string[] expectedWarningStrings = null,
string[] expectedErrorStrings = null)
{
if (config == null)
{
config = new SandboxConfiguration {
FileAccessIgnoreCodeCoverage = true, FailUnexpectedFileAccesses = true
};
}
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var fileContentTable = FileContentTable.CreateNew(LoggingContext);
// have to force the config for truncation
config.OutputReportingMode = OutputReportingMode.FullOutputOnWarningOrError;
bool expectSuccess = expectedErrorStrings == null && expectedWarningStrings == null;
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
{
string currentCodeFolder = Path.GetDirectoryName(AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly()));
Contract.Assume(currentCodeFolder != null);
string executable = Path.Combine(currentCodeFolder, DetourTestFolder, "DetoursTests.exe");
string workingDirectory = tempFiles.GetUniqueDirectory();
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
XAssert.IsTrue(File.Exists(executable), "Could not find the test file: " + executable);
FileArtifact executableFileArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
var extraUntrackedScopes = new List <AbsolutePath>();
var dependencies = new List <FileArtifact> {
executableFileArtifact
};
if (extraDependencies != null)
{
foreach (string file in extraDependencies)
{
string filePath = Path.Combine(workingDirectory, file);
AbsolutePath path = AbsolutePath.Create(pathTable, filePath);
if (readsAndWritesDirectories)
{
Directory.CreateDirectory(filePath);
// We don't support directories as inputs in BuildXL yet.
extraUntrackedScopes.Add(path);
}
else
{
File.WriteAllText(filePath, "Definitely a file");
FileArtifact fileArtifact = FileArtifact.CreateSourceFile(path);
dependencies.Add(fileArtifact);
}
}
}
var outputs = new List <FileArtifactWithAttributes>();
if (extraOutputs != null)
{
foreach (string file in extraOutputs)
{
string filePath = Path.Combine(workingDirectory, file);
AbsolutePath path = AbsolutePath.Create(pathTable, filePath);
if (readsAndWritesDirectories)
{
// We don't support directory outputs in BuildXL at the moment, so e.g. deleting a directory needs to be untracked.
extraUntrackedScopes.Add(path);
}
else if (untrackedOutputs)
{
extraUntrackedScopes.Add(path);
}
else
{
FileArtifact fileArtifact = FileArtifact.CreateSourceFile(path).CreateNextWrittenVersion();
outputs.Add(fileArtifact.WithAttributes());
}
}
}
var tempDirectory = tempFiles.GetUniqueDirectory();
var environmentVariables = new List <EnvironmentVariable>();
var environmentValue = new PipDataBuilder(pathTable.StringTable);
var tempPath = AbsolutePath.Create(pathTable, tempDirectory);
environmentValue.Add(tempPath);
environmentVariables.Add(new EnvironmentVariable(StringId.Create(pathTable.StringTable, "TMP"), environmentValue.ToPipData(" ", PipDataFragmentEscaping.NoEscaping)));
environmentVariables.Add(new EnvironmentVariable(StringId.Create(pathTable.StringTable, "TEMP"), environmentValue.ToPipData(" ", PipDataFragmentEscaping.NoEscaping)));
var untrackedPaths = CmdHelper.GetCmdDependencies(pathTable);
var untrackedScopes = extraUntrackedScopes.Concat(CmdHelper.GetCmdDependencyScopes(pathTable).Concat(new[] { tempPath })).Distinct();
var pip = new Process(
executableFileArtifact,
workingDirectoryAbsolutePath,
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.NoEscaping, commandPrefix + functionName + "Logging"),
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .From(environmentVariables),
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pathTable),
null,
null,
ReadOnlyArray <FileArtifact> .From(dependencies),
ReadOnlyArray <FileArtifactWithAttributes> .From(outputs),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(untrackedPaths),
ReadOnlyArray <AbsolutePath> .From(untrackedScopes),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
if (expectSuccess)
{
await AssertProcessSucceedsAsync(
context,
config,
pip);
}
else
{
await AssertProcessCompletesWithStatusAsync(
SandboxedProcessPipExecutionStatus.ExecutionFailed,
context,
config,
pip,
null);
}
}
int expectedErrorCount = 0;
int expectedWarningCount = 0;
IEnumerable <string> requiredLogMessageSubstrings = new string[] { };
if (expectedErrorStrings != null)
{
expectedErrorCount = expectedErrorStrings.Count();
requiredLogMessageSubstrings = requiredLogMessageSubstrings.Concat(expectedErrorStrings);
}
if (expectedWarningStrings != null)
{
expectedWarningCount = expectedWarningStrings.Count();
requiredLogMessageSubstrings = requiredLogMessageSubstrings.Concat(expectedWarningStrings);
}
SetExpectedFailures(expectedErrorCount, expectedWarningCount, requiredLogMessageSubstrings.ToArray());
}
/// <summary>
/// Adds a fake copy file pip that produces to the given destination path.
/// </summary>
public CopyFile AddCopyFilePip(FileArtifact source, FileArtifact destination)
{
Contract.Requires(source != null);
Contract.Requires(destination != null);
var copyFile = new CopyFile(source, destination, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(m_context));
copyFile.PipId = AllocateNextPipId();
m_pips.Add(copyFile.PipId, copyFile);
m_pathProducers.Add(destination.Path, copyFile);
return(copyFile);
}
/// <summary>
/// Adds a fake write file pip that produces to the given destination path.
/// </summary>
public WriteFile AddWriteFilePip(AbsolutePath destinationPath)
{
Contract.Requires(destinationPath != null);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationPath).CreateNextWrittenVersion();
PipData contents = PipDataBuilder.CreatePipData(m_context.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "content");
var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(m_context));
writeFile.PipId = AllocateNextPipId();
m_pips.Add(writeFile.PipId, writeFile);
m_pathProducers.Add(destinationArtifact, writeFile);
return(writeFile);
}
