/// <nodoc />
public SandboxedProcessPipExecutionResult(
SandboxedProcessPipExecutionStatus status,
SortedReadOnlyArray<ObservedFileAccess, ObservedFileAccessExpandedPathComparer> observedFileAccesses,
IReadOnlyDictionary<AbsolutePath, IReadOnlyCollection<FileArtifactWithAttributes>> sharedDynamicDirectoryWriteAccesses,
Tuple<AbsolutePath, Encoding> encodedStandardOutput,
Tuple<AbsolutePath, Encoding> encodedStandardError,
int numberOfWarnings,
FileAccessReportingContext unexpectedFileAccesses,
ProcessTimes primaryProcessTimes,
JobObject.AccountingInformation? jobAccountingInformation,
int numberOfProcessLaunchRetries,
int exitCode,
long sandboxPrepMs,
long processSandboxedProcessResultMs,
long processStartTime,
IReadOnlyList<ReportedFileAccess> allReportedFileAccesses,
IReadOnlyList<ProcessDetouringStatusData> detouringStatuses,
long maxDetoursHeapSize,
ContainerConfiguration containerConfiguration,
Dictionary<string, int> pipProperties,
bool timedOut,
CancellationReason cancellationReason = CancellationReason.None)
{
Contract.Requires(
(status == SandboxedProcessPipExecutionStatus.PreparationFailed ||
status == SandboxedProcessPipExecutionStatus.ShouldBeRetriedDueToUserSpecifiedExitCode ||
status == SandboxedProcessPipExecutionStatus.Canceled) ||
observedFileAccesses.IsValid);
Contract.Requires(
(status == SandboxedProcessPipExecutionStatus.PreparationFailed || status == SandboxedProcessPipExecutionStatus.ShouldBeRetriedDueToUserSpecifiedExitCode ||
status == SandboxedProcessPipExecutionStatus.Canceled) ||
unexpectedFileAccesses != null);
Contract.Requires((status == SandboxedProcessPipExecutionStatus.PreparationFailed || status == SandboxedProcessPipExecutionStatus.Canceled) || primaryProcessTimes != null);
Contract.Requires(encodedStandardOutput == null || (encodedStandardOutput.Item1.IsValid && encodedStandardOutput.Item2 != null));
Contract.Requires(encodedStandardError == null || (encodedStandardError.Item1.IsValid && encodedStandardError.Item2 != null));
Contract.Requires(numberOfWarnings >= 0);
Contract.Requires(containerConfiguration != null);
Status = status;
ObservedFileAccesses = observedFileAccesses;
UnexpectedFileAccesses = unexpectedFileAccesses;
EncodedStandardOutput = encodedStandardOutput;
EncodedStandardError = encodedStandardError;
NumberOfWarnings = numberOfWarnings;
PrimaryProcessTimes = primaryProcessTimes;
JobAccountingInformation = jobAccountingInformation;
NumberOfProcessLaunchRetries = numberOfProcessLaunchRetries;
ExitCode = exitCode;
SandboxPrepMs = sandboxPrepMs;
ProcessSandboxedProcessResultMs = processSandboxedProcessResultMs;
ProcessStartTimeMs = processStartTime;
AllReportedFileAccesses = allReportedFileAccesses;
DetouringStatuses = detouringStatuses;
MaxDetoursHeapSizeInBytes = maxDetoursHeapSize;
SharedDynamicDirectoryWriteAccesses = sharedDynamicDirectoryWriteAccesses;
ContainerConfiguration = containerConfiguration;
PipProperties = pipProperties;
TimedOut = timedOut;
CancellationReason = cancellationReason;
}
/// <nodoc />
private static DirectoryArtifact SealDirectoryPartialOrFull(
PipConstructionHelper pipConstructionHelper,
AbsolutePath directoryRoot,
SealDirectoryKind kind,
FileArtifact[] contents,
string[] tags = null,
string description = null,
string[] patterns = null)
{
if (!pipConstructionHelper.TrySealDirectory(
directoryRoot,
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(
contents,
OrdinalFileArtifactComparer.Instance),
CollectionUtilities.EmptySortedReadOnlyArray <DirectoryArtifact, OrdinalDirectoryArtifactComparer>(OrdinalDirectoryArtifactComparer.Instance),
kind,
tags,
description,
patterns,
out var sealedDirectory
))
{
throw new InvalidOperationException();
}
return(sealedDirectory);
}
/// <nodoc />
public SandboxedProcessPipExecutionResult(
SandboxedProcessPipExecutionStatus status,
SortedReadOnlyArray <ObservedFileAccess, ObservedFileAccessExpandedPathComparer> observedFileAccesses,
IReadOnlyDictionary <AbsolutePath, IReadOnlyCollection <FileArtifactWithAttributes> > sharedDynamicDirectoryWriteAccesses,
Tuple <AbsolutePath, Encoding> encodedStandardOutput,
Tuple <AbsolutePath, Encoding> encodedStandardError,
int numberOfWarnings,
FileAccessReportingContext unexpectedFileAccesses,
ProcessTimes primaryProcessTimes,
JobObject.AccountingInformation?jobAccountingInformation,
int exitCode,
long sandboxPrepMs,
long processSandboxedProcessResultMs,
long processStartTime,
IReadOnlyList <ReportedFileAccess> allReportedFileAccesses,
IReadOnlyList <ProcessDetouringStatusData> detouringStatuses,
long maxDetoursHeapSize,
ContainerConfiguration containerConfiguration,
Dictionary <string, int> pipProperties,
bool timedOut,
IReadOnlySet <AbsolutePath> createdDirectories,
RetryInfo retryInfo = null)
{
Contract.Requires(!ProcessCompletedExecution(status) || observedFileAccesses.IsValid);
Contract.Requires(!ProcessCompletedExecution(status) || unexpectedFileAccesses != null);
Contract.Requires(!ProcessCompletedExecution(status) || primaryProcessTimes != null);
Contract.Requires(encodedStandardOutput == null || (encodedStandardOutput.Item1.IsValid && encodedStandardOutput.Item2 != null));
Contract.Requires(encodedStandardError == null || (encodedStandardError.Item1.IsValid && encodedStandardError.Item2 != null));
Contract.Requires(numberOfWarnings >= 0);
Contract.Requires(containerConfiguration != null);
Contract.Requires(retryInfo == null || status != SandboxedProcessPipExecutionStatus.Succeeded);
// Protect against invalid combinations of RetryLocation and RetryReason
Contract.Requires(!retryInfo.CanBeRetriedInlineOrFalseIfNull() || retryInfo.RetryReason != RetryReason.ResourceExhaustion);
Contract.Requires(!retryInfo.CanBeRetriedInlineOrFalseIfNull() || retryInfo.RetryReason != RetryReason.ProcessStartFailure);
Contract.Requires(!retryInfo.CanBeRetriedInlineOrFalseIfNull() || retryInfo.RetryReason != RetryReason.TempDirectoryCleanupFailure);
Contract.Requires(!retryInfo.CanBeRetriedInlineOrFalseIfNull() || retryInfo.RetryReason != RetryReason.StoppedWorker);
Status = status;
ObservedFileAccesses = observedFileAccesses;
UnexpectedFileAccesses = unexpectedFileAccesses;
EncodedStandardOutput = encodedStandardOutput;
EncodedStandardError = encodedStandardError;
NumberOfWarnings = numberOfWarnings;
PrimaryProcessTimes = primaryProcessTimes;
JobAccountingInformation = jobAccountingInformation;
ExitCode = exitCode;
SandboxPrepMs = sandboxPrepMs;
ProcessSandboxedProcessResultMs = processSandboxedProcessResultMs;
ProcessStartTimeMs = processStartTime;
AllReportedFileAccesses = allReportedFileAccesses;
DetouringStatuses = detouringStatuses;
MaxDetoursHeapSizeInBytes = maxDetoursHeapSize;
SharedDynamicDirectoryWriteAccesses = sharedDynamicDirectoryWriteAccesses;
ContainerConfiguration = containerConfiguration;
PipProperties = pipProperties;
TimedOut = timedOut;
RetryInfo = retryInfo;
CreatedDirectories = createdDirectories ?? CollectionUtilities.EmptySet <AbsolutePath>();
}
/// <summary>
/// Creates a pip representing the hashing of the given source artifact.
/// </summary>
public SealDirectory(
AbsolutePath directoryRoot,
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> contents,
SortedReadOnlyArray <DirectoryArtifact, OrdinalDirectoryArtifactComparer> outputDirectoryContents,
SealDirectoryKind kind,
PipProvenance provenance,
ReadOnlyArray <StringId> tags,
ReadOnlyArray <StringId> patterns,
bool scrub = false)
{
Contract.Requires(directoryRoot.IsValid);
Contract.Requires(contents.IsValid);
Contract.Requires(outputDirectoryContents.IsValid);
Contract.Requires(tags.IsValid);
Contract.Requires(provenance != null);
Contract.Requires(!(patterns.IsValid && patterns.Length != 0) || kind.IsSourceSeal(), "If patterns are provided, it must be a source seal directory");
Contract.Requires(!scrub || kind.IsFull(), "Only scrub fully seal directory");
Contract.Requires(outputDirectoryContents.Length == 0 || kind.IsFull(), "Output directory content is only available for fully sealed directories");
Provenance = provenance;
DirectoryRoot = directoryRoot;
Contents = contents;
OutputDirectoryContents = outputDirectoryContents;
Kind = kind;
Tags = tags;
Scrub = scrub;
Patterns = patterns;
// Initialization required before this pip is usable or serializable.
m_partialSealId = null;
}
public DirectoryArtifact SealDirectory(
AbsolutePath directoryRoot,
IReadOnlyList <FileArtifact> contents,
SealDirectoryKind kind,
string[] tags = null,
string description = null,
string[] patterns = null,
bool scrub = false)
{
if (!PipConstructionHelper.TrySealDirectory(
directoryRoot,
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(
contents,
OrdinalFileArtifactComparer.Instance),
CollectionUtilities.EmptySortedReadOnlyArray <DirectoryArtifact, OrdinalDirectoryArtifactComparer>(OrdinalDirectoryArtifactComparer.Instance),
kind,
tags,
description,
patterns,
out DirectoryArtifact sealedDirectory,
scrub))
{
throw new BuildXLTestException("Failed to add sealDirectory pip");
}
return(sealedDirectory);
}
public void IsValidTests()
{
var testArray = new SortedReadOnlyArray <string, StringComparer>();
Assert.False(testArray.IsValid);
Assert.False(testArray.BaseArray.IsValid);
testArray = SortedReadOnlyArray <string, StringComparer> .CloneAndSort(new string[0], StringComparer.Ordinal);
Assert.True(testArray.IsValid);
Assert.True(testArray.BaseArray.IsValid);
testArray = CollectionUtilities.EmptySortedReadOnlyArray <string, StringComparer>(StringComparer.Ordinal);
Assert.True(testArray.IsValid);
Assert.True(testArray.BaseArray.IsValid);
bool exceptionThrown = false;
try
{
testArray = SortedReadOnlyArray <string, StringComparer> .CloneAndSort(null, StringComparer.Ordinal);
}
#pragma warning disable ERP022 // Unobserved exception in generic exception handler
catch
{
exceptionThrown = true;
}
#pragma warning restore ERP022 // Unobserved exception in generic exception handler
Assert.True(exceptionThrown);
}
private static bool TryScheduleSealDirectory(TestEnv env, AbsolutePath path, SealDirectoryKind partial, FileArtifact[] contents)
{
var pip = new SealDirectory(
path,
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(contents, OrdinalFileArtifactComparer.Instance),
kind: partial,
provenance: env.CreatePipProvenance(StringId.Invalid),
tags: ReadOnlyArray <StringId> .Empty,
patterns: ReadOnlyArray <StringId> .Empty);
DirectoryArtifact artifact = env.PipGraph.AddSealDirectory(pip, PipId.Invalid);
bool succeeded = artifact.IsValid;
if (succeeded)
{
FileArtifact[] actualContents = GetSealedDirectoryContents(env, artifact);
XAssert.AreEqual(contents.Length, actualContents.Length, "Wrong number of contents sealed");
for (int i = 0; i < contents.Length; i++)
{
XAssert.IsTrue(contents[i] == actualContents[i], "Content artifact at position {0} mismatched", i);
}
}
return(succeeded);
}
private static EvaluationResult SealDirectoryHelper(Context context, ModuleLiteral env, EvaluationStackFrame args, SealDirectoryKind sealDirectoryKind)
{
AbsolutePath path = Args.AsPath(args, 0, false);
ArrayLiteral contents = Args.AsArrayLiteral(args, 1);
var tags = Args.AsStringArrayOptional(args, 2);
var description = Args.AsStringOptional(args, 3);
// Only do scrub for fully seal directory
var scrub = sealDirectoryKind.IsFull() ? Args.AsBoolOptional(args, 4) : false;
var fileContents = new FileArtifact[contents.Length];
for (int i = 0; i < contents.Length; ++i)
{
fileContents[i] = Converter.ExpectFile(contents[i], strict: false, context: new ConversionContext(pos: i, objectCtx: contents));
}
var sortedFileContents = SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(fileContents, OrdinalFileArtifactComparer.Instance);
DirectoryArtifact sealedDirectoryArtifact;
if (!context.GetPipConstructionHelper().TrySealDirectory(path, sortedFileContents, sealDirectoryKind, tags, description, null, out sealedDirectoryArtifact, scrub))
{
// Error has been logged
return(EvaluationResult.Error);
}
var result = new StaticDirectory(sealedDirectoryArtifact, sealDirectoryKind, sortedFileContents.WithCompatibleComparer(OrdinalPathOnlyFileArtifactComparer.Instance));
return(new EvaluationResult(result));
}
private async Task <CasHash> AddPathSet(ICacheSession session, params string[] thePaths)
{
var pathTable = new PathTable();
ObservedPathEntry[] paths = new ObservedPathEntry[thePaths.Length];
for (int i = 0; i < thePaths.Length; i++)
{
AbsolutePath absPath = AbsolutePath.Create(pathTable, thePaths[i]);
paths[i] = new ObservedPathEntry(absPath, false, false, false, null, false);
}
var emptyObservedAccessFileNames = SortedReadOnlyArray <StringId, CaseInsensitiveStringIdComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <StringId> .Empty,
new CaseInsensitiveStringIdComparer(pathTable.StringTable));
ObservedPathSet pathSet = new ObservedPathSet(
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <ObservedPathEntry> .FromWithoutCopy(paths),
new ObservedPathEntryExpandedPathComparer(pathTable.ExpandedPathComparer)),
emptyObservedAccessFileNames,
null);
using (var pathSetBuffer = new MemoryStream())
{
using (var writer = new BuildXLWriter(stream: pathSetBuffer, debug: false, leaveOpen: true, logStats: false))
{
pathSet.Serialize(pathTable, writer, preserveCasing: false);
}
pathSetBuffer.Seek(0, SeekOrigin.Begin);
// Must await such that the dispose of the MemoryStream is only after the write completes
return(await session.AddToCasAsync(pathSetBuffer).SuccessAsync());
}
}
/// <summary>
/// Writes a SortedReadOnlyArray
/// </summary>
public void Write <TValue, TComparer>(SortedReadOnlyArray <TValue, TComparer> value, Action <BuildXLWriter, TValue> writer)
where TComparer : class, IComparer <TValue>
{
Contract.Requires(value.IsValid);
Contract.RequiresNotNull(writer);
Start <SortedReadOnlyArray <TValue, TComparer> >();
Write(value.BaseArray, writer);
End();
}
private EvaluationResult EnsureContents(Context context, StaticDirectory receiver, EvaluationResult arg, EvaluationStackFrame captures)
{
// Check the kind. For now this only works on Full and Partial Sealed directories
// This needs to be extended (just like getFile and getFiles) to opaque and sourcesealed directories.
var stringTable = context.FrontEndContext.StringTable;
var pathTable = context.FrontEndContext.PathTable;
switch (receiver.SealDirectoryKind)
{
case SealDirectoryKind.Full:
case SealDirectoryKind.Partial:
// Supported since we have static directory.
break;
default:
// For the other types we will need to schedule a pip in the graph that actually validates at runtime the file is there
// either on disk for sourcesealed, or in the opaque collection by using FileContentManager.ListSealedDirectoryContents
throw new DirectoryNotSupportedException(receiver.Root.Path.ToString(pathTable));
}
var obj = Converter.ExpectObjectLiteral(arg);
var subFolder = Converter.ExtractRelativePath(obj, m_subFolder);
var filterPath = receiver.Root.Path.Combine(pathTable, subFolder);
var fileContents = new List <FileArtifact>();
foreach (var sealedFile in receiver.Contents)
{
if (sealedFile.Path.IsWithin(pathTable, filterPath))
{
fileContents.Add(sealedFile);
}
}
var sortedFileContents = SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(fileContents, OrdinalFileArtifactComparer.Instance);
if (!context.GetPipConstructionHelper().TrySealDirectory(
directoryRoot: filterPath,
contents: sortedFileContents,
outputDirectorycontents: CollectionUtilities.EmptySortedReadOnlyArray <DirectoryArtifact, OrdinalDirectoryArtifactComparer>(OrdinalDirectoryArtifactComparer.Instance),
kind: SealDirectoryKind.Partial,
tags: null,
description: null,
patterns: null,
sealedDirectory: out var sealedDirectoryArtifact,
scrub: false))
{
// Error has been logged
return(EvaluationResult.Error);
}
var result = new StaticDirectory(sealedDirectoryArtifact, SealDirectoryKind.Partial, sortedFileContents.WithCompatibleComparer(OrdinalPathOnlyFileArtifactComparer.Instance));
return(EvaluationResult.Create(result));
}
/// <nodoc />
private async Task <EvaluationResult> ValidateAndStoreIncrementalExtractState(PipConstructionHelper pipConstructionHelper, DownloadData downloadData)
{
var archive = downloadData.DownloadedFilePath.ToString(m_context.PathTable);
var target = downloadData.ContentsFolder.Path.ToString(m_context.PathTable);
try
{
var allFiles = new List <FileArtifact>();
var enumeratResult = FileUtilities.EnumerateDirectoryEntries(target, true, "*",
(dir, fileName, attributes) =>
{
if ((attributes & FileAttributes.Directory) == 0)
{
var filePath = Path.Combine(dir, fileName);
allFiles.Add(FileArtifact.CreateSourceFile(AbsolutePath.Create(m_context.PathTable, filePath)));
}
});
if (!enumeratResult.Succeeded)
{
var error = new Win32Exception(enumeratResult.NativeErrorCode);
m_logger.ErrorListingPackageContents(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, error.Message);
return(EvaluationResult.Error);
}
if (allFiles.Count == 0)
{
m_logger.ErrorListingPackageContents(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, "file list is empty");
return(EvaluationResult.Error);
}
var sortedFiles = SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(
allFiles,
OrdinalFileArtifactComparer.Instance);
var hashes = new Dictionary <AbsolutePath, ContentHash>();
foreach (var file in allFiles)
{
var hash = await GetContentHashAsync(file);
hashes.Add(file.Path, hash);
}
var incrementalState = new ExtractIncrementalState(downloadData, sortedFiles, hashes);
await incrementalState.SaveAsync(m_context);
return(SealDirectory(pipConstructionHelper, downloadData, downloadData.ContentsFolder, sortedFiles));
}
catch (Exception e)
when(e is BuildXLException || e is IOException || e is UnauthorizedAccessException)
{
m_logger.ErrorExtractingArchive(m_context.LoggingContext, downloadData.Settings.ModuleName, archive, target, e.Message);
return(EvaluationResult.Error);
}
}
/// <summary>
/// Constructs a path set from already-sorted paths. The array may contain duplicates.
/// </summary>
public ObservedPathSet(
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> paths,
SortedReadOnlyArray <StringId, CaseInsensitiveStringIdComparer> observedAccessedFileNames,
[CanBeNull] UnsafeOptions unsafeOptions)
{
Contract.Requires(paths.IsValid);
Paths = paths;
ObservedAccessedFileNames = observedAccessedFileNames;
UnsafeOptions = unsafeOptions ?? UnsafeOptions.SafeValues;
}
/// <summary>
/// Creates the representation of an already-sealed directory.
/// </summary>
public StaticDirectory(DirectoryArtifact root, SealDirectoryKind kind, SortedReadOnlyArray <FileArtifact, OrdinalPathOnlyFileArtifactComparer> contents)
{
Contract.Requires(root.IsValid);
// Opaques and shared opaques should always have empty content
Contract.Requires(!kind.IsDynamicKind() || contents.Length == 0);
Root = root;
Contents = contents;
SealDirectoryKind = kind;
Kind = GetScriptRepresentation(SealDirectoryKind);
}
private EvaluationResult SealDirectoryHelper(Context context, ModuleLiteral env, EvaluationStackFrame args, SealDirectoryKind sealDirectoryKind)
{
AbsolutePath path;
ArrayLiteral contents;
string[] tags;
string description;
bool scrub;
if (args.Length > 0 && args[0].Value is ObjectLiteral)
{
var obj = Args.AsObjectLiteral(args, 0);
var directory = Converter.ExtractDirectory(obj, m_sealRoot, allowUndefined: false);
path = directory.Path;
contents = Converter.ExtractArrayLiteral(obj, m_sealFiles, allowUndefined: false);
tags = Converter.ExtractStringArray(obj, m_sealTags, allowUndefined: true);
description = Converter.ExtractString(obj, m_sealDescription, allowUndefined: true);
scrub = sealDirectoryKind.IsFull()
? Converter.ExtractOptionalBoolean(obj, m_sealScrub) ?? false
: false;
}
else
{
path = Args.AsPath(args, 0, false);
contents = Args.AsArrayLiteral(args, 1);
tags = Args.AsStringArrayOptional(args, 2);
description = Args.AsStringOptional(args, 3);
// Only do scrub for fully seal directory
scrub = sealDirectoryKind.IsFull() ? Args.AsBoolOptional(args, 4) : false;
}
var fileContents = new FileArtifact[contents.Length];
for (int i = 0; i < contents.Length; ++i)
{
fileContents[i] = Converter.ExpectFile(contents[i], strict: false, context: new ConversionContext(pos: i, objectCtx: contents));
}
var sortedFileContents = SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(fileContents, OrdinalFileArtifactComparer.Instance);
DirectoryArtifact sealedDirectoryArtifact;
if (!context.GetPipConstructionHelper().TrySealDirectory(path, sortedFileContents, sealDirectoryKind, tags, description, null, out sealedDirectoryArtifact, scrub))
{
// Error has been logged
return(EvaluationResult.Error);
}
var result = new StaticDirectory(sealedDirectoryArtifact, sealDirectoryKind, sortedFileContents.WithCompatibleComparer(OrdinalPathOnlyFileArtifactComparer.Instance));
return(new EvaluationResult(result));
}
/// <summary>
/// Creates an instance of <see cref="ObservedGraphInputs"/>.
/// </summary>
public ObservedGraphInputs(
SortedReadOnlyArray <GraphPathInput, GraphPathInput.ByPathAndKindComparer> pathInputs,
SortedReadOnlyArray <EnvironmentVariableInput, EnvironmentVariableInput.ByNameComparer> environmentVariableInputs,
SortedReadOnlyArray <MountInput, MountInput.ByNameComparer> mountInputs)
{
Contract.Requires(pathInputs.IsValid);
Contract.Requires(environmentVariableInputs.IsValid);
Contract.Requires(mountInputs.IsValid);
PathInputs = pathInputs;
EnvironmentVariableInputs = environmentVariableInputs;
MountInputs = mountInputs;
}
/// <summary>
/// Reads a SortedReadOnlyArray
/// </summary>
public SortedReadOnlyArray <TValue, TComparer> ReadSortedReadOnlyArray <TValue, TComparer>(
Func <BuildXLReader, TValue> reader,
TComparer comparer)
where TComparer : class, IComparer <TValue>
{
Contract.Requires(reader != null);
Contract.Requires(comparer != null);
Start <SortedReadOnlyArray <TValue, TComparer> >();
ReadOnlyArray <TValue> array = ReadReadOnlyArray(reader);
End();
return(SortedReadOnlyArray <TValue, TComparer> .FromSortedArrayUnsafe(array, comparer));
}
private static SealDirectory CreateSeal(TestEnv env, AbsolutePath path, bool partial, FileArtifact[] contents)
{
var seal = new SealDirectory(
path,
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(contents, OrdinalFileArtifactComparer.Instance),
kind: partial ? SealDirectoryKind.Partial : SealDirectoryKind.Full,
provenance: env.CreatePipProvenance(StringId.Invalid),
tags: ReadOnlyArray <StringId> .Empty,
patterns: ReadOnlyArray <StringId> .Empty);
env.PipTable.Add(((PipGraph.Builder)(env.PipGraph)).MutableDataflowGraph.CreateNode().Value, seal);
return(seal);
}
public static ObservedPathSet CreatePathSet(PathTable pathTable, params string[] paths)
{
AbsolutePath[] pathIds = new AbsolutePath[paths.Length];
for (int i = 0; i < paths.Length; i++)
{
pathIds[i] = AbsolutePath.Create(pathTable, paths[i]);
}
SortedReadOnlyArray <AbsolutePath, PathTable.ExpandedAbsolutePathComparer> sortedPathIds =
SortedReadOnlyArray <AbsolutePath, PathTable.ExpandedAbsolutePathComparer> .SortUnsafe(
pathIds,
pathTable.ExpandedPathComparer);
return(CreatePathSet(pathTable, pathIds));
}
private static ObservedInputProcessingResult CreateResult(PathTable pathTable, params ObservedInput[] inputs)
{
var sorted = SortedReadOnlyArray <ObservedInput, ObservedInputExpandedPathComparer> .SortUnsafe(
inputs,
new ObservedInputExpandedPathComparer(pathTable.ExpandedPathComparer));
var emptyObservedAccessFileNames = SortedReadOnlyArray <StringId, CaseInsensitiveStringIdComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <StringId> .Empty,
new CaseInsensitiveStringIdComparer(pathTable.StringTable));
return(ObservedInputProcessingResult.CreateForSuccess(
sorted,
emptyObservedAccessFileNames,
dynamicObservations: ReadOnlyArray <(AbsolutePath, DynamicObservationKind)> .Empty,
allowedUndeclaredSourceReads: CollectionUtilities.EmptySet <AbsolutePath>()));
}
/// <nodoc />
public bool TrySealDirectory(
AbsolutePath directoryRoot,
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> contents,
SortedReadOnlyArray <DirectoryArtifact, OrdinalDirectoryArtifactComparer> outputDirectorycontents,
SealDirectoryKind kind,
string[] tags,
string description,
string[] patterns,
out DirectoryArtifact sealedDirectory,
bool scrub = false)
{
Contract.Requires(directoryRoot.IsValid);
Contract.Requires(contents.IsValid);
Contract.Requires(outputDirectorycontents.IsValid);
PipData usage = PipDataBuilder.CreatePipData(Context.StringTable, string.Empty, PipDataFragmentEscaping.NoEscaping, description != null
? new PipDataAtom[] { description }
: new PipDataAtom[] { "'", directoryRoot, "' [", contents.Length.ToString(CultureInfo.InvariantCulture), " files - ",
outputDirectorycontents.Length.ToString(CultureInfo.InvariantCulture), " output directories]" });
var pip = new SealDirectory(
directoryRoot,
contents,
outputDirectorycontents,
kind,
CreatePipProvenance(usage),
ToStringIds(tags),
ToStringIds(patterns),
scrub);
if (PipGraph != null)
{
sealedDirectory = PipGraph.AddSealDirectory(pip, GetValuePipId());
if (!sealedDirectory.IsValid)
{
return(false);
}
}
else
{
sealedDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(directoryRoot);
}
return(true);
}
public static ObservedPathSet CreatePathSet(PathTable pathTable, params AbsolutePath[] paths)
{
ObservedPathEntry[] entries = paths.Select(p => new ObservedPathEntry(p, false, false, false, null, false)).ToArray();
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> sortedPathIds =
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> .SortUnsafe(
entries,
new ObservedPathEntryExpandedPathComparer(pathTable.ExpandedPathComparer));
var emptyObservedAccessFileNames = SortedReadOnlyArray <StringId, CaseInsensitiveStringIdComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <StringId> .Empty,
new CaseInsensitiveStringIdComparer(pathTable.StringTable));
return(new ObservedPathSet(
sortedPathIds,
emptyObservedAccessFileNames,
new UnsafeOptions(UnsafeOptions.SafeConfigurationValues, new PreserveOutputsInfo(ContentHashingUtilities.CreateRandom(), 0))));
}
private ProcessStrongFingerprintComputationData Convert(ProcessStrongFingerprintComputationData computation)
{
if (AreGraphsSame)
{
return(computation);
}
var pathSet = new ObservedPathSet(
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> .FromSortedArrayUnsafe(
Convert(computation.PathEntries, this, (i, me) => me.Convert(i)),
new ObservedPathEntryExpandedPathComparer(OldModel.PathTable.ExpandedPathComparer)),
computation.PathSet.ObservedAccessedFileNames,
computation.PathSet.UnsafeOptions);
return(computation.UnsafeOverride(
pathSet,
Convert(computation.ObservedInputs, this, (i, me) => me.Convert(i))));
}
public static List <TValue2> RemoveDuplicates <TValue, TValue2, TComparer>(
SortedReadOnlyArray <TValue, TComparer> values,
Func <TValue, TValue, bool> comparer,
Func <TValue, TValue2> selector)
where TComparer : class, IComparer <TValue>
{
var uniqueEntries = new List <TValue2>();
for (int i = 0; i < values.Length; i++)
{
while (i + 1 < values.Length && comparer(values[i + 1], values[i]))
{
i++;
}
uniqueEntries.Add(selector(values[i]));
}
return(uniqueEntries);
}
/// <summary>
/// Associates a directory artifact with its constituent file artifacts. The contents must be set before the directory artifact may
/// be used by a pip (see <see cref="ListSealedDirectoryContents"/>).
/// </summary>
public void SetSealedDirectoryContents(DirectoryArtifact directory, params FileArtifact[] artifactsInDirectory)
{
Contract.Requires(directory.IsValid);
Contract.Requires(artifactsInDirectory != null);
FileArtifact[] artifacts = artifactsInDirectory.ToArray();
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> sortedArtifacts = SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .SortUnsafe(
artifacts,
OrdinalFileArtifactComparer.Instance);
bool added = m_knownSealedDirectoryContents.TryAdd(directory, sortedArtifacts);
Contract.Assume(added);
foreach (FileArtifact artifact in artifacts)
{
Contract.Assume(artifact.Path.IsWithin(Context.PathTable, directory.Path));
m_knownSealedArtifacts[artifact.Path] = artifact;
}
}
/// <inheritdoc />
public SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> ListSealedDirectoryContents(DirectoryArtifact directoryArtifact)
{
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> artifacts;
if (!m_knownSealedDirectoryContents.TryGetValue(directoryArtifact, out artifacts))
{
if (m_allowUnspecifiedSealedDirectories)
{
FileArtifact[] sourceArtifactsUnderSealRoot =
Directory.EnumerateFiles(directoryArtifact.Path.ToString(Context.PathTable), "*", SearchOption.AllDirectories)
.Select(p => FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, p)))
.ToArray();
return(SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .SortUnsafe(
sourceArtifactsUnderSealRoot,
OrdinalFileArtifactComparer.Instance));
}
else
{
Contract.Assume(false, "Unknown directory artifact for path " + directoryArtifact.Path.ToString(Context.PathTable));
}
}
return(artifacts);
}
/// <inheritdoc/>
public SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> ListSealDirectoryContents(DirectoryArtifact directory)
{
return(SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(m_inputContent, OrdinalFileArtifactComparer.Instance));
}
/// <inheritdoc/>
public SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> ListSealDirectoryContents(DirectoryArtifact directory, out IReadOnlySet <AbsolutePath> temporaryFiles)
{
temporaryFiles = CollectionUtilities.EmptySet <AbsolutePath>();
return(SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .CloneAndSort(m_inputContent, OrdinalFileArtifactComparer.Instance));
}
private EvaluationResult SealDirectory(PipConstructionHelper pipConstructionHelper, DownloadData downloadData, DirectoryArtifact directory, SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> files)
{
if (!pipConstructionHelper.TrySealDirectory(
directory,
files,
CollectionUtilities.EmptySortedReadOnlyArray <DirectoryArtifact, OrdinalDirectoryArtifactComparer>(OrdinalDirectoryArtifactComparer.Instance),
Pips.Operations.SealDirectoryKind.Partial,
null,
null,
null,
out var directoryArtifact)
)
{
return(EvaluationResult.Error);
}
var staticDirectory = new StaticDirectory(directoryArtifact, Pips.Operations.SealDirectoryKind.Partial, files.WithCompatibleComparer(OrdinalPathOnlyFileArtifactComparer.Instance));
return(new EvaluationResult(staticDirectory));
}
/// <summary>
/// Extracts a file into a folder with in manifest based incrementality.
/// </summary>
internal async Task <EvaluationResult> PerformExtractOrIncrementalCheckAsync(DownloadData downloadData)
{
if (m_context.CancellationToken.IsCancellationRequested)
{
return(EvaluationResult.Canceled);
}
// Ensure file is downloaded
var extractedFileResult = await DownloadFile(downloadData);
if (extractedFileResult.IsErrorValue)
{
return(extractedFileResult);
}
var extractedFile = (FileArtifact)extractedFileResult.Value;
var moduleDescriptor = m_workspaceResolver.GetModuleDescriptor(downloadData);
var pipConstructionHelper = PipConstructionHelper.Create(
m_context,
m_frontEndHost.Engine.Layout.ObjectDirectory,
m_frontEndHost.Engine.Layout.RedirectedDirectory,
m_frontEndHost.Engine.Layout.TempDirectory,
m_frontEndHost.PipGraph,
moduleDescriptor.Id,
moduleDescriptor.Name,
RelativePath.Create(downloadData.ModuleSpecFile.GetName(m_context.PathTable)),
FullSymbol.Create(m_context.SymbolTable, "extracted"),
new LocationData(downloadData.ModuleSpecFile, 0, 0),
m_context.QualifierTable.EmptyQualifierId);
// When we don't have to extract we'll expose the downloaded file in the contents.
if (downloadData.Settings.ArchiveType == DownloadArchiveType.File)
{
return(SealDirectory(
pipConstructionHelper,
downloadData,
DirectoryArtifact.CreateWithZeroPartialSealId(downloadData.DownloadedFilePath.GetParent(m_context.PathTable)),
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <FileArtifact> .FromWithoutCopy(new[] { extractedFile }),
OrdinalFileArtifactComparer.Instance)));
}
Statistics.Extractions.Total.Increment();
using (Statistics.Extractions.UpToDateCheckDuration.Start(downloadData.Settings.Url))
{
var result = await CheckIfExtractIsNeededAsync(pipConstructionHelper, downloadData);
if (result.IsErrorValue)
{
Statistics.Extractions.Failures.Increment();
}
if (result != EvaluationResult.Continue)
{
Statistics.Extractions.SkippedDueToManifest.Increment();
return(result);
}
}
using (Statistics.Extractions.Duration.Start(downloadData.Settings.Url))
{
try
{
if (!await Task.Run(
() => TryExtractToDisk(downloadData),
m_context.CancellationToken))
{
Statistics.Extractions.Failures.Increment();
return(EvaluationResult.Error);
}
}
catch (TaskCanceledException)
{
return(EvaluationResult.Canceled);
}
}
using (Statistics.Extractions.UpToDateCheckDuration.Start(downloadData.Settings.Url))
{
var result = await ValidateAndStoreIncrementalExtractState(pipConstructionHelper, downloadData);
if (result.IsErrorValue)
{
Statistics.Extractions.Failures.Increment();
}
return(result);
}
}
