public SpecFileRef(PipExecutionContext context, AbsolutePath path)
: base(context, path)
{
}
internal NotifyMasterExecutionLogTarget(uint workerId, IMasterClient masterClient, PipExecutionContext context, Guid logId, int lastStaticAbsolutePathIndex, DistributionServices services)
: this(new NotifyStream(workerId, masterClient, services), context, logId, lastStaticAbsolutePathIndex)
{
}
/// <summary>
/// Initiates the task to load the fingerprint store that will be used for cache miss analysis
/// </summary>
public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration))
{
var option = configuration.Logging.CacheMissAnalysisOption;
if (option.Mode == CacheMissMode.Disabled)
{
return(null);
}
Possible <FingerprintStore> possibleStore;
if (option.Mode == CacheMissMode.Local)
{
possibleStore = FingerprintStore.CreateSnapshot(logTarget.FingerprintStore, loggingContext);
}
else
{
string path = null;
if (option.Mode == CacheMissMode.CustomPath)
{
path = option.CustomPath.ToString(context.PathTable);
}
else
{
Contract.Assert(option.Mode == CacheMissMode.Remote);
foreach (var key in option.Keys)
{
var cacheSavePath = configuration.Logging.EngineCacheLogDirectory
.Combine(context.PathTable, Scheduler.FingerprintStoreDirectory + "_" + key);
#pragma warning disable AsyncFixer02 // This should explicitly happen synchronously since it interacts with the PathTable and StringTable
var result = cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, context.PathTable, key, configuration.Schedule.EnvironmentFingerprint).Result;
#pragma warning restore AsyncFixer02
if (result.Succeeded && result.Result)
{
path = cacheSavePath.ToString(context.PathTable);
break;
}
}
if (string.IsNullOrEmpty(path))
{
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}"));
return(null);
}
}
// Unblock caller
// WARNING: The rest can simultenously happen with saving the graph files to disk.
// We should not create any paths or strings by using PathTable and StringTable.
await Task.Yield();
possibleStore = FingerprintStore.Open(path, readOnly: true);
}
if (possibleStore.Succeeded)
{
Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory);
return(new RuntimeCacheMissAnalyzer(logTarget, loggingContext, context, possibleStore.Result, graph, runnablePipPerformance));
}
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}"));
return(null);
}
}
/// <summary>
/// Constructs an Ide Generator and generates the files
/// </summary>
public static bool Generate(PipExecutionContext pipContext, PipGraph pipGraph, IReadonlyDirectedGraph scheduledGraph, AbsolutePath configFilePath, IIdeConfiguration ideConfig)
{
var generator = new IdeGenerator(pipContext, pipGraph, scheduledGraph, configFilePath, ideConfig);
return(generator.Generate());
}
/// <nodoc />
public SchedulerServiceManager(PipGraph pipGraph, PipExecutionContext context)
{
m_pipGraph = pipGraph;
m_context = context;
}
public DirectoryRef(PipExecutionContext context, AbsolutePath path)
: base(context, path)
{
}
public PathPipDataEntry(PipExecutionContext context, AbsolutePath p)
{
P = new PathRef(context, p);
}
private void WriteGraphNodeEntry(
Pip pip,
IPipScheduleTraversal graph,
PipExecutionContext context,
Dictionary <DirectoryArtifact, int> directoryIds,
Dictionary <PipId, ProcessExecutionMonitoringReportedEventData> directoryInputContent,
Dictionary <PipId, PipExecutionDirectoryOutputs> directoryOutputContent)
{
Contract.Requires(pip != null);
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WriteStartObject();
{
WriteStaticPipDetails(pip, context);
{
// We get nice space savings by omitting dependsOn when it is empty (think HashSourceFile and WriteFile pips).
bool writtenHeader = false;
foreach (var dependency in graph.RetrievePipImmediateDependencies(pip))
{
if (!writtenHeader)
{
m_writer.WritePropertyName("dependsOn");
m_writer.WriteStartArray();
writtenHeader = true;
}
if (IncludePip(dependency.PipType))
{
m_writer.WriteValue(dependency.PipId.Value);
}
}
if (writtenHeader)
{
m_writer.WriteEndArray();
}
writtenHeader = false;
PipArtifacts.ForEachInput(pip, dependency =>
{
int dependencyId;
if (dependency.IsFile)
{
if (!m_fileIds.TryGetValue(dependency.FileArtifact.Path, out dependencyId))
{
Contract.Assume(false, "Expected file artifact already written (dependency of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
else
{
if (!directoryIds.TryGetValue(dependency.DirectoryArtifact, out dependencyId))
{
Contract.Assume(false, "Expected directory artifact already written (input of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
if (!writtenHeader)
{
m_writer.WritePropertyName("consumes");
m_writer.WriteStartArray();
writtenHeader = true;
}
m_writer.WriteValue(dependencyId);
return(true);
}, includeLazyInputs: true);
// Write reads from shared opaque directories
if (directoryInputContent.TryGetValue(pip.PipId, out var pipInput))
{
foreach (var input in pipInput.ReportedFileAccesses)
{
int dependencyId;
if (!m_fileIds.TryGetValue(input.ManifestPath, out dependencyId))
{
// Ignore unrecognized reads (these are usually directories / files that aren't produced, so we don't have their ID)
continue;
}
m_writer.WriteValue(dependencyId);
}
}
if (writtenHeader)
{
m_writer.WriteEndArray();
}
}
{
m_writer.WritePropertyName("produces");
m_writer.WriteStartArray();
SortedSet <int> reads = new SortedSet <int>();
PipArtifacts.ForEachOutput(pip, dependency =>
{
int dependencyId;
if (dependency.IsFile)
{
if (!m_fileIds.TryGetValue(dependency.FileArtifact.Path, out dependencyId))
{
Contract.Assume(false, "Expected file artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
else
{
if (!directoryIds.TryGetValue(dependency.DirectoryArtifact, out dependencyId))
{
Contract.Assume(false, "Expected directory artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
reads.Add(dependencyId);
return(true);
}, includeUncacheable: true);
// Write outputs into shared opaque directories
if (directoryOutputContent.TryGetValue(pip.PipId, out var pipOutput))
{
foreach (var output in pipOutput.DirectoryOutputs)
{
DirectoryArtifact dir = output.directoryArtifact;
var content = output.fileArtifactArray;
foreach (var file in content)
{
int dependencyId;
if (!m_fileIds.TryGetValue(file.Path, out dependencyId))
{
Contract.Assume(false, "Expected file artifact not found in fileId table " + file.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
reads.Add(dependencyId);
}
}
}
foreach (int dependencyId in reads)
{
m_writer.WriteValue(dependencyId);
}
m_writer.WriteEndArray();
}
}
m_writer.WriteEndObject();
}
private void WriteStaticPipDetails(Pip pip, PipExecutionContext context)
{
Contract.Requires(pip != null);
Contract.Requires(pip.PipType != PipType.HashSourceFile);
m_writer.WritePropertyName("pipId");
m_writer.WriteValue(pip.PipId.Value);
if (pip.SemiStableHash != 0)
{
m_writer.WritePropertyName("stableId");
// The X16 format agrees with e.g. PipDC41BEE27D4187E2 which is the most user-visible presently.
// See Pip.GetDescription
m_writer.WriteValue(pip.SemiStableHash.ToString("X16", CultureInfo.InvariantCulture));
}
if (pip.Provenance != null)
{
m_writer.WritePropertyName("provenance");
m_writer.WriteStartObject();
{
m_writer.WritePropertyName("value");
m_writer.WriteValue(pip.Provenance.OutputValueSymbol.ToString(context.SymbolTable));
m_writer.WritePropertyName("spec");
m_writer.WriteValue(pip.Provenance.Token.Path.ToString(context.PathTable));
}
m_writer.WriteEndObject();
}
m_writer.WritePropertyName("type");
m_writer.WriteValue(pip.PipType.ToString());
m_writer.WritePropertyName("description");
m_writer.WriteValue(pip.GetDescription(context));
var process = pip as Process;
if (process != null)
{
m_writer.WritePropertyName("exe");
m_writer.WriteValue(process.GetToolName(context.PathTable).ToString(context.StringTable));
if (process.Semaphores != null && process.Semaphores.Length > 0)
{
m_writer.WritePropertyName("semaphores");
m_writer.WriteStartArray();
foreach (ProcessSemaphoreInfo semaphore in process.Semaphores)
{
m_writer.WriteStartObject();
m_writer.WritePropertyName("name");
m_writer.WriteValue(semaphore.Name.ToString(context.StringTable));
m_writer.WritePropertyName("value");
m_writer.WriteValue(semaphore.Value);
m_writer.WritePropertyName("limit");
m_writer.WriteValue(semaphore.Limit);
m_writer.WriteEndObject();
}
m_writer.WriteEndArray();
}
}
}
private void WriteGraphNodeEntry(
Pip pip,
IPipScheduleTraversal graph,
PipExecutionContext context,
Dictionary <DirectoryArtifact, int> directoryIds)
{
Contract.Requires(pip != null);
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WriteStartObject();
{
WriteStaticPipDetails(pip, context);
{
// We get nice space savings by omitting dependsOn when it is empty (think HashSourceFile and WriteFile pips).
bool writtenHeader = false;
foreach (var dependency in graph.RetrievePipImmediateDependencies(pip))
{
if (!writtenHeader)
{
m_writer.WritePropertyName("dependsOn");
m_writer.WriteStartArray();
writtenHeader = true;
}
if (IncludePip(dependency.PipType))
{
m_writer.WriteValue(dependency.PipId.Value);
}
}
if (writtenHeader)
{
m_writer.WriteEndArray();
}
writtenHeader = false;
PipArtifacts.ForEachInput(pip, dependency =>
{
int dependencyId;
if (dependency.IsFile)
{
if (!m_fileIds.TryGetValue(dependency.FileArtifact, out dependencyId))
{
Contract.Assume(false, "Expected file artifact already written (dependency of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
else
{
if (!directoryIds.TryGetValue(dependency.DirectoryArtifact, out dependencyId))
{
Contract.Assume(false, "Expected directory artifact already written (input of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
if (!writtenHeader)
{
m_writer.WritePropertyName("consumes");
m_writer.WriteStartArray();
writtenHeader = true;
}
m_writer.WriteValue(dependencyId);
return(true);
}, includeLazyInputs: true);
if (writtenHeader)
{
m_writer.WriteEndArray();
}
}
{
m_writer.WritePropertyName("produces");
m_writer.WriteStartArray();
PipArtifacts.ForEachOutput(pip, dependency =>
{
int dependencyId;
if (dependency.IsFile)
{
if (!m_fileIds.TryGetValue(dependency.FileArtifact, out dependencyId))
{
Contract.Assume(false, "Expected file artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
else
{
if (!directoryIds.TryGetValue(dependency.DirectoryArtifact, out dependencyId))
{
Contract.Assume(false, "Expected directory artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
m_writer.WriteValue(dependencyId);
return(true);
}, includeUncacheable: true);
m_writer.WriteEndArray();
}
}
m_writer.WriteEndObject();
}
/// <summary>
/// Save the graph. This method produces the whole file content. It must be called only once.
/// </summary>
public void SaveGraphData(
IPipScheduleTraversal graph,
PipExecutionContext context,
IEnumerable <PipExecutionPerformanceEventData> executionData,
IEnumerable <string> workers,
IEnumerable <KeyValuePair <FileArtifact, FileContentInfo> > fileData,
Dictionary <PipId, ProcessExecutionMonitoringReportedEventData> directoryInputContent,
Dictionary <PipId, PipExecutionDirectoryOutputs> directoryOutputContent)
{
Contract.Requires(graph != null);
Contract.Requires(context != null);
Contract.Assume(m_fileIds.Count == 0);
// We have transitioned to the Exporting state.
// Observed execution info may get queued up for when we begin draining the queue (below).
// Don't overlap pip ids with file/directory ids
int nextId = graph.PipCount + 1;
var directoryIds = new Dictionary <DirectoryArtifact, int>(capacity: 1000);
m_writer.WriteStartObject(); // Outermost object
WritePreamble();
m_writer.WritePropertyName("artifacts");
m_writer.WriteStartArray();
// To save some work on pip deserialization (we are pathologically visiting every pip in the pip table),
// we hold pips alive between the various passes.
var pips = new List <Pip>();
{
var directoryPips = new List <SealDirectory>();
// TODO: This is pretty expensive, as it loads all pips in memory
foreach (Pip pip in graph.RetrieveScheduledPips())
{
pips.Add(pip);
// We retrieve outputs rather than inputs since we do not expect duplicate outputs among pips
// (not true with inputs). This means the ID assignment work is linear in the number of artifacts.
PipArtifacts.ForEachOutput(pip, output =>
{
if (output.IsFile)
{
AssignFileIdAndWriteFileEntry(output.FileArtifact, context, ref nextId);
}
return(true);
},
includeUncacheable: true);
// SealDirectory pips are the only ones with directory outputs. As part of the artifact entry for the directory output,
// we want to capture the membership of the directory. This means we want to have assigned IDs to all member files before
// writing out that list (for easier parsing). So, we defer writing out seal directory pips until all file artifacts have been visited.
if (pip.PipType == PipType.SealDirectory)
{
var directoryPip = (SealDirectory)pip;
Contract.Assume(directoryPip.IsInitialized);
directoryPips.Add(directoryPip);
}
// Record files that are written into SharedOpaqueDirectories
if (directoryOutputContent.TryGetValue(pip.PipId, out var pipOutput))
{
foreach (var output in pipOutput.DirectoryOutputs)
{
DirectoryArtifact dir = output.directoryArtifact;
var content = output.fileArtifactArray;
foreach (var file in content)
{
AssignFileIdAndWriteFileEntry(file, context, ref nextId);
}
}
}
}
foreach (SealDirectory directoryPip in directoryPips)
{
AssignDirectoryIdAndWriteDirectoryEntry(directoryPip.Directory, context, directoryPip.Contents, directoryIds, ref nextId);
}
}
m_writer.WriteEndArray();
m_writer.WritePropertyName("graph");
m_writer.WriteStartArray();
{
// Note we are using the 'pips' list captured above rather than possibly deserializing pips again.
foreach (Pip pip in pips)
{
if (!IncludePip(pip.PipType))
{
continue;
}
WriteGraphNodeEntry(pip, graph, context, directoryIds, directoryInputContent, directoryOutputContent);
}
}
m_writer.WriteEndArray();
// Avoid holding a reference to this map if it isn't later needed by the special file details exporter
if (fileData == null)
{
m_fileIds = null;
}
if (executionData != null)
{
m_writer.WritePropertyName("execution");
m_writer.WriteStartArray();
// Begin draining the execution entry queue, now that we are in an allowed state for that.
// Iteration will complete only after Finish() is called, which marks the queue as complete (no new items).
foreach (var pipWithPerf in executionData)
{
WriteExecutionEntry(pipWithPerf.PipId, pipWithPerf.ExecutionPerformance);
}
// End the execution: array
m_writer.WriteEndArray();
}
if (workers != null)
{
ExportWorkerDetails(workers);
}
if (fileData != null)
{
ExportFileDetails(fileData);
}
// End the outermost object
m_writer.WriteEndObject();
m_writer.Flush();
}
public WhitelistFileMonitoringViolationAnalyzer(LoggingContext loggingContext, PipExecutionContext context, PipGraph pipGraph, WhitelistAnalyzer whitelistAnalyzer)
: base(loggingContext, context, pipGraph, new QueryableFileContentManager(whitelistAnalyzer.OutputDirectoryContent), validateDistribution: false, ignoreDynamicWritesOnAbsentProbes: false, unexpectedFileAccessesAsErrors: true)
{
m_whitelistAnalyzer = whitelistAnalyzer;
}
public CacheCoreArtifactContentCacheTests(ITestOutputHelper output)
: base(output)
{
Context = BuildXLContext.CreateInstanceForTesting();
ContentCache = new CacheCoreArtifactContentCache(Session, rootTranslator: null);
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
IKextConnection sandboxedKextConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(config != null);
LoggingContext = loggingContext;
Context = context;
// Ensure paths visible when debugging
PathTable.DebugPathTable = Context.PathTable;
Configuration = config;
PipTable = pipTable;
PathExpander = semanticPathExpander ?? SemanticPathExpander.Default;
ContentFingerprinter = new PipContentFingerprinter(
Context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null),
pathExpander: PathExpander,
pipDataLookup: pipDataLookup);
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = ipcProvider ?? IpcFactory.GetProvider();
FileContentTable = fileContentTable ?? FileContentTable.CreateNew();
Cache = pipCache;
FileAccessWhitelist = fileAccessWhitelist;
m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories;
m_sandboxedKextConnection = sandboxedKextConnection;
if (Cache == null)
{
Cache = InMemoryCacheFactory.Create();
}
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker);
PipGraphView = new TestPipGraphFilesystemView(Context.PathTable);
m_operationTracker = new OperationTracker(loggingContext);
var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore);
var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed;
if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled)
{
preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString());
}
State = new PipExecutionState(
config,
cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: ExecutionLogRecorder,
fileSystemView: fileSystemView,
fileContentManager: GetFileContentManager(),
directoryMembershipFinterprinterRuleSet: null,
unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: preserveOutputsSalt,
serviceManager: new DummyServiceManager());
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
/// <summary>
/// Creates a new PipConstructionHelper
/// </summary>
/// <remarks>
/// Ideally this function would take ModuleId, FullSymbol QualifierId and compute uniqueOutputLocation itself. Unfortunately today the data is not yet
/// exposed via IPipGraph, therefore the responsibility is on the call site for now.
/// </remarks>
public static PipConstructionHelper Create(
PipExecutionContext context,
AbsolutePath objectRoot,
AbsolutePath redirectedRoot,
AbsolutePath tempRoot,
IPipGraph pipGraph,
ModuleId moduleId,
string moduleName,
RelativePath specRelativePath,
FullSymbol symbol,
LocationData thunkLocation,
QualifierId qualifierId)
{
var stringTable = context.StringTable;
var pathTable = context.PathTable;
// We have to manually compute the pipPipUniqueString here, Ideally we pass PackageId, SpecFile, FullSymbol and qualiferId and have it computed inside, but the IPipGraph does not allow querying it for now.
string hashString;
long semiStableHashSeed = 0;
using (var builderWrapper = Pools.GetStringBuilder())
{
var builder = builderWrapper.Instance;
builder.Append(moduleName);
builder.Append('/');
semiStableHashSeed = HashCodeHelper.GetOrdinalHashCode64(moduleName);
if (specRelativePath.IsValid)
{
string specPath = specRelativePath.ToString(stringTable);
builder.Append(specPath);
builder.Append('/');
semiStableHashSeed = HashCodeHelper.Combine(semiStableHashSeed, HashCodeHelper.GetOrdinalHashCode64(specPath));
}
var symbolName = symbol.ToStringAsCharArray(context.SymbolTable);
builder.Append(symbolName);
builder.Append('/');
semiStableHashSeed = HashCodeHelper.Combine(semiStableHashSeed, HashCodeHelper.GetOrdinalHashCode64(symbolName));
var qualifierDisplayValue = context.QualifierTable.GetCanonicalDisplayString(qualifierId);
builder.Append(qualifierDisplayValue);
semiStableHashSeed = HashCodeHelper.Combine(semiStableHashSeed, HashCodeHelper.GetOrdinalHashCode64(qualifierDisplayValue));
var pipPipUniqueString = builder.ToString();
hashString = Hash(pipPipUniqueString);
}
var pipRelativePath = RelativePath.Create(
PathAtom.Create(stringTable, hashString.Substring(0, 1)),
PathAtom.Create(stringTable, hashString.Substring(1, 1)),
PathAtom.Create(stringTable, hashString.Substring(2)));
var valuePip = new ValuePip(symbol, qualifierId, thunkLocation);
return(new PipConstructionHelper(
context,
objectRoot,
redirectedRoot,
tempRoot,
pipGraph,
moduleId,
moduleName,
valuePip,
pipRelativePath,
semiStableHashSeed));
}
/// <summary>
/// Merges the outputs of the given redirected process to its original location based on configured policies
/// </summary>
public Task <bool> MergeOutputsIfNeededAsync(Process process, ContainerConfiguration containerConfiguration, PipExecutionContext pipExecutionContext, IReadOnlyDictionary <AbsolutePath, IReadOnlyCollection <AbsolutePath> > sharedDynamicWrites)
{
Contract.Requires(containerConfiguration != null);
Contract.Requires(pipExecutionContext != null);
if (!process.NeedsToRunInContainer)
{
return(Task.FromResult(true));
}
return(Task.Run(
() =>
{
try
{
using (var createdDirectoriesWrapper = Pools.AbsolutePathSetPool.GetInstance())
{
HashSet <AbsolutePath> cretedDirectories = createdDirectoriesWrapper.Instance;
var result = HardlinkAllDeclaredOutputs(process, containerConfiguration, pipExecutionContext, cretedDirectories);
if (result != MergeResult.Success)
{
return false;
}
result = HardlinkOpaqueDirectories(process, containerConfiguration, pipExecutionContext, sharedDynamicWrites, cretedDirectories);
if (result != MergeResult.Success)
{
return false;
}
}
}
catch (BuildXLException ex)
{
// The operation above may throw, and in that case we don't want to propagate the exception, but grab the error message
// and interpret it as a merge failure
Tracing.Logger.Log.FailedToMergeOutputsToOriginalLocation(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), ex.Message);
return false;
}
return true;
}));
}
/// <summary>
/// Constructor
/// </summary>
protected RemoteWorkerBase(uint workerId, string name, PipExecutionContext context)
: base(workerId, name, context)
{
}
private MergeResult HardlinkAllDeclaredOutputs(Process process, ContainerConfiguration containerConfiguration, PipExecutionContext pipExecutionContext, HashSet <AbsolutePath> createdDirectories)
{
if (!process.ContainerIsolationLevel.IsolateOutputFiles())
{
return(MergeResult.Success);
}
foreach (FileArtifactWithAttributes fileOutput in process.FileOutputs)
{
// If the output is not there, just continue. Checking all outputs are present already happened, so this means an optional output.
// If the output is a WCI reparse point or tombstone file, continue as well. This means the output was either not actually generated (but a file with that name was read) or the file was generated and later deleted.
string sourcePath = GetRedirectedOutputPathForDeclaredOutput(containerConfiguration, pipExecutionContext, fileOutput.Path);
var sourcePathExists = FileUtilities.Exists(sourcePath);
if (!sourcePathExists || FileUtilities.IsWciReparseArtifact(sourcePath))
{
// If the declared output is a WCI artifact, that means it is not a real output. If we reached this point it is because the sandbox process executor determined this output
// was an optional one, and the content was actually not produced. Since when storing the process into the cache we will try to use the redirected file to retrieve its content,
// make sure we remove the WCI artifact, so the file actually appears as absent
// Observe this is not really needed for the case of opaque directories, since the content that gets reported is based on the existence of the file in its original location, which
// is not hardlinked if the file is a WCI artifact. We could remove them in the opaque case as well, just for a matter of symetry, but since this may have perf implications, we are not
// doing that right now.
if (sourcePathExists)
{
FileUtilities.DeleteFile(sourcePath);
}
continue;
}
string destinationPath = fileOutput.Path.ToString(m_pathTable);
var mergeResult = TryCreateHardlinkForOutput(fileOutput.Path.Expand(m_pathTable), fileOutput.RewriteCount, sourcePath, process, pipExecutionContext, createdDirectories);
if (mergeResult != MergeResult.Success)
{
return(mergeResult);
}
}
return(MergeResult.Success);
}
public ParallelBinaryLogReader(string logFilename, PipExecutionContext context)
: base(File.Open(logFilename, FileMode.Open, FileAccess.Read, FileShare.Read), context, true)
{
m_logFilename = logFilename;
}
private MergeResult HardlinkOpaqueDirectories(
Process process,
ContainerConfiguration containerConfiguration,
PipExecutionContext pipExecutionContext,
IReadOnlyDictionary <AbsolutePath, IReadOnlyCollection <AbsolutePath> > sharedDynamicWrites,
HashSet <AbsolutePath> createdDirectories)
{
bool isolateSharedOpaques = process.ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories();
bool isolateExclusiveOpaques = process.ContainerIsolationLevel.IsolateExclusiveOpaqueOutputDirectories();
// Shortcut the iteration of output directories are not isolated at all
if (!isolateExclusiveOpaques && !isolateSharedOpaques)
{
return(MergeResult.Success);
}
foreach (DirectoryArtifact directoryOutput in process.DirectoryOutputs)
{
if (directoryOutput.IsSharedOpaque && isolateSharedOpaques)
{
AbsolutePath redirectedDirectory = GetRedirectedDirectoryForOutputContainer(containerConfiguration, directoryOutput.Path).Path;
// Here we don't need to check for WCI reparse points. We know those outputs are there based on what detours is saying.
// However, we need to check for tombstone files: those represent deleted files in some scenarios (for example, moves), so we don't hardlink them
var sharedOpaqueContent = sharedDynamicWrites[directoryOutput.Path];
foreach (AbsolutePath sharedOpaqueFile in sharedOpaqueContent)
{
string sourcePath = sharedOpaqueFile.Relocate(m_pathTable, directoryOutput.Path, redirectedDirectory).ToString(m_pathTable);
// The file may not exist because the pip could have created it but later deleted it, or be a tombstone file
if (!FileUtilities.Exists(sourcePath) || FileUtilities.IsWciTombstoneFile(sourcePath))
{
continue;
}
ExpandedAbsolutePath destinationPath = sharedOpaqueFile.Expand(m_pathTable);
// Files in an opaque always have rewrite count 1
var result = TryCreateHardlinkForOutput(destinationPath, rewriteCount: 1, sourcePath, process, pipExecutionContext, createdDirectories);
if (result != MergeResult.Success)
{
return(result);
}
}
}
else if (!directoryOutput.IsSharedOpaque && isolateExclusiveOpaques)
{
// We need to enumerate to discover the content of an exclusive opaque, and also skip the potential reparse artifacts
// TODO: Enumeration will happen again when the file content manager tries to discover the content of the exclusive opaque. Consider doing this only once instead.
// An output directory should only have one redirected path
ExpandedAbsolutePath redirectedDirectory = containerConfiguration.OriginalDirectories[directoryOutput.Path].Single();
foreach (string exclusiveOpaqueFile in Directory.EnumerateFiles(redirectedDirectory.ExpandedPath, "*", SearchOption.AllDirectories))
{
if (FileUtilities.IsWciReparseArtifact(exclusiveOpaqueFile))
{
continue;
}
AbsolutePath exclusiveOpaqueFilePath = AbsolutePath.Create(m_pathTable, exclusiveOpaqueFile);
AbsolutePath outputFile = exclusiveOpaqueFilePath.Relocate(m_pathTable, redirectedDirectory.Path, directoryOutput.Path);
// Files in an opaque always have rewrite count 1
var result = TryCreateHardlinkForOutput(outputFile.Expand(m_pathTable), rewriteCount: 1, exclusiveOpaqueFile, process, pipExecutionContext, createdDirectories);
if (result != MergeResult.Success)
{
return(result);
}
}
}
}
return(MergeResult.Success);
}
/// <summary>
/// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>.
/// </summary>
/// <returns>
/// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to
/// a <see cref="Tracing.FingerprintStore"/> at the provided directory;
/// otherwise, null.
/// </returns>
public static FingerprintStoreExecutionLogTarget Create(
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
LoggingContext loggingContext,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
CounterCollection <FingerprintStoreCounters> counters,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null,
FingerprintStoreTestHooks testHooks = null)
{
var fingerprintStorePathString = configuration.Layout.FingerprintStoreDirectory.ToString(context.PathTable);
var cacheLookupFingerprintStorePathString = configuration.Logging.CacheLookupFingerprintStoreLogDirectory.ToString(context.PathTable);
try
{
FileUtilities.CreateDirectoryWithRetry(fingerprintStorePathString);
}
catch (BuildXLException ex)
{
Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message);
throw new BuildXLException("Unable to create fingerprint store directory: ", ex);
}
var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0);
var possibleExecutionStore = FingerprintStore.Open(
fingerprintStorePathString,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
counters: counters,
testHooks: testHooks);
Possible <FingerprintStore> possibleCacheLookupStore = new Failure <string>("No attempt to create a cache lookup fingerprint store yet.");
if (configuration.Logging.FingerprintStoreMode != FingerprintStoreMode.ExecutionFingerprintsOnly)
{
try
{
FileUtilities.CreateDirectoryWithRetry(cacheLookupFingerprintStorePathString);
}
catch (BuildXLException ex)
{
Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message);
throw new BuildXLException("Unable to create fingerprint store directory: ", ex);
}
possibleCacheLookupStore = FingerprintStore.Open(
cacheLookupFingerprintStorePathString,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
counters: counters,
testHooks: testHooks);
}
if (possibleExecutionStore.Succeeded &&
(possibleCacheLookupStore.Succeeded || configuration.Logging.FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly))
{
return(new FingerprintStoreExecutionLogTarget(
loggingContext,
context,
pipTable,
pipContentFingerprinter,
possibleExecutionStore.Result,
possibleCacheLookupStore.Succeeded ? possibleCacheLookupStore.Result : null,
configuration,
cache,
graph,
counters,
runnablePipPerformance));
}
else
{
if (!possibleExecutionStore.Succeeded)
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleExecutionStore.Failure.DescribeIncludingInnerFailures());
}
if (!possibleCacheLookupStore.Succeeded)
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleCacheLookupStore.Failure.DescribeIncludingInnerFailures());
}
}
return(null);
}
private MergeResult TryCreateHardlinkForOutput(ExpandedAbsolutePath fileOutput, int rewriteCount, string sourcePath, Process process, PipExecutionContext pipExecutionContext, HashSet <AbsolutePath> createdDirectories)
{
if (!CanMerge(fileOutput.ExpandedPath, rewriteCount, process, pipExecutionContext, out bool isDisallowed, out bool shouldDelete))
{
if (isDisallowed)
{
Tracing.Logger.Log.DisallowedDoubleWriteOnMerge(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), fileOutput.ExpandedPath, sourcePath);
return(MergeResult.DisallowedDoubleWrite);
}
// The file cannot be merged, but no merging is allowed
return(MergeResult.Success);
}
if (shouldDelete)
{
FileUtilities.DeleteFile(fileOutput.ExpandedPath, waitUntilDeletionFinished: true);
}
else
{
// If we need to delete the target file, the directory is already created. So only do this otherwise.
// Make sure the destination directory exists before trying to create the hardlink
var containingDirectory = fileOutput.Path.GetParent(m_pathTable);
if (!createdDirectories.Contains(containingDirectory))
{
string destinationDirectory = containingDirectory.ToString(m_pathTable);
if (!FileUtilities.DirectoryExistsNoFollow(destinationDirectory))
{
FileUtilities.CreateDirectory(destinationDirectory);
}
createdDirectories.Add(containingDirectory);
}
}
var createHardlinkStatus = FileUtilities.TryCreateHardLink(fileOutput.ExpandedPath, sourcePath);
if (createHardlinkStatus != CreateHardLinkStatus.Success)
{
// Maybe somebody else raced and successfully created the hardlink since the last time we checked,
// so let's check again before failing
if (!CanMerge(fileOutput.ExpandedPath, rewriteCount, process, pipExecutionContext, out isDisallowed, out bool _))
{
if (isDisallowed)
{
Tracing.Logger.Log.DisallowedDoubleWriteOnMerge(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), fileOutput.ExpandedPath, sourcePath);
return(MergeResult.DisallowedDoubleWrite);
}
// The file cannot be merged, but no merging is allowed
return(MergeResult.Success);
}
Tracing.Logger.Log.FailedToCreateHardlinkOnMerge(m_loggingContext, process.SemiStableHash, process.GetDescription(pipExecutionContext), fileOutput.ExpandedPath, sourcePath, createHardlinkStatus.ToString());
return(MergeResult.Failure);
}
return(MergeResult.Success);
}
/// <summary>
/// Constructs an Ide Generator from the BuildXL.Execution.Analyzer
/// </summary>
public IdeGenerator(PipExecutionContext pipContext, PipGraph pipGraph, IReadonlyDirectedGraph scheduledGraph, AbsolutePath configFilePath, IIdeConfiguration ideConfig)
{
m_context = new Context(pipContext, pipGraph, scheduledGraph, configFilePath, ideConfig);
}
private string GetRedirectedOutputPathForDeclaredOutput(ContainerConfiguration containerConfiguration, PipExecutionContext pipExecutionContext, AbsolutePath fileOutput)
{
AbsolutePath containingDirectory = fileOutput.GetParent(m_pathTable);
var redirectedDirectory = GetRedirectedDirectoryForOutputContainer(containerConfiguration, containingDirectory);
var sourcePath = Path.Combine(redirectedDirectory.ExpandedPath, fileOutput.GetName(m_pathTable).ToString(pipExecutionContext.StringTable));
return(sourcePath);
}
public AllowlistFileMonitoringViolationAnalyzer(LoggingContext loggingContext, PipExecutionContext context, PipGraph pipGraph, AllowlistAnalyzer allowlistAnalyzer)
: base(loggingContext, context, pipGraph, new QueryableFileContentManager(allowlistAnalyzer.OutputDirectoryContent), validateDistribution: false, ignoreDynamicWritesOnAbsentProbes: Utilities.Configuration.DynamicWriteOnAbsentProbePolicy.IgnoreNothing, unexpectedFileAccessesAsErrors: true)
{
m_allowlistAnalyzer = allowlistAnalyzer;
}
/// <nodoc />
public InMemoryArtifactContentCache(PipExecutionContext context)
: this(context, new object(), new Dictionary <ContentHash, CacheEntry>())
{
Contract.Requires(context != null);
}
private NotifyMasterExecutionLogTarget(NotifyStream stream, PipExecutionContext context, Guid logId, int lastStaticAbsolutePathIndex)
: base(new BinaryLogger(stream, context, logId, lastStaticAbsolutePathIndex, closeStreamOnDispose: true, onEventWritten: () => stream.FlushIfNeeded()), closeLogFileOnDispose: true)
{
}
/// <summary>
/// Wraps the content cache's content for use with another execution context
/// </summary>
public InMemoryArtifactContentCache WrapForContext(PipExecutionContext context)
{
return(new InMemoryArtifactContentCache(context, m_lock, m_content));
}
/// <summary>
/// Internal constructor
/// </summary>
/// <param name="moduleId">The id of the module</param>
/// <param name="moduleNameId">The StringId of the name of the module</param>
/// <param name="moduleLocationId">The AbsolutePath of the location of the module definition file</param>
/// <param name="context">Used to convert the StringId and AbsolutePath values into strings</param>
internal ModuleDescriptor(int moduleId, StringId moduleNameId, AbsolutePath moduleLocationId, PipExecutionContext context)
{
ModuleId = moduleId;
m_moduleNameId = moduleNameId;
m_moduleLocationId = moduleLocationId;
m_context = context;
}
/// <summary>
/// Loads configured symlink definitions (if not already loaded)
/// Stores to cache for use by workers in distributed build
/// Eagerly creates symlinks if lazy symlink creation is disabled
/// </summary>
public static async Task <Possible <SymlinkDefinitions> > TryPrepareSymlinkDefinitionsAsync(
LoggingContext loggingContext,
GraphReuseResult reuseResult,
IConfiguration configuration,
MasterService masterService,
CacheInitializationTask cacheInitializerTask,
PipExecutionContext context,
ITempCleaner tempDirectoryCleaner = null)
{
var pathTable = context.PathTable;
bool isDistributedMaster = configuration.Distribution.BuildRole == DistributedBuildRoles.Master;
Possible <SymlinkDefinitions> maybeSymlinkDefinitions = new Possible <SymlinkDefinitions>((SymlinkDefinitions)null);
if (reuseResult?.IsFullReuse == true)
{
maybeSymlinkDefinitions = reuseResult.EngineSchedule.Scheduler.SymlinkDefinitions;
}
else if (configuration.Layout.SymlinkDefinitionFile.IsValid)
{
var symlinkFilePath = configuration.Layout.SymlinkDefinitionFile.ToString(pathTable);
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Loading symlink file from location '{symlinkFilePath}'."));
maybeSymlinkDefinitions = await SymlinkDefinitions.TryLoadAsync(
loggingContext,
pathTable,
symlinkFilePath,
symlinksDebugPath : configuration.Logging.LogsDirectory.Combine(pathTable, "DebugSymlinksDefinitions.log").ToString(pathTable),
tempDirectoryCleaner : tempDirectoryCleaner);
}
if (!maybeSymlinkDefinitions.Succeeded || maybeSymlinkDefinitions.Result == null)
{
return(maybeSymlinkDefinitions);
}
// Need to store symlinks to cache for workers
if (configuration.Distribution.BuildRole == DistributedBuildRoles.Master)
{
var possibleCacheInitializer = await cacheInitializerTask;
if (!possibleCacheInitializer.Succeeded)
{
return(possibleCacheInitializer.Failure);
}
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Storing symlink file for use by workers."));
var symlinkFile = configuration.Layout.SymlinkDefinitionFile.Expand(pathTable);
var possibleStore = await TryStoreToCacheAsync(
loggingContext,
cache : possibleCacheInitializer.Result.CreateCacheForContext().ArtifactContentCache,
symlinkFile : symlinkFile);
if (!possibleStore.Succeeded)
{
return(possibleStore.Failure);
}
masterService.SymlinkFileContentHash = possibleStore.Result;
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Stored symlink file for use by workers."));
}
if (!configuration.Schedule.UnsafeLazySymlinkCreation || configuration.Engine.PopulateSymlinkDirectories.Count != 0)
{
// Symlink definition file is defined, and BuildXL intends to create it eagerly.
// At this point master and worker should have had its symlink definition file, if specified.
if (!FileContentManager.CreateSymlinkEagerly(loggingContext, configuration, pathTable, maybeSymlinkDefinitions.Result, context.CancellationToken))
{
return(new Failure <string>("Failed eagerly creating symlinks"));
}
}
return(maybeSymlinkDefinitions);
}
