private async Task <CasHash> AddPathSet(ICacheSession session, params string[] thePaths)
{
var pathTable = new PathTable();
ObservedPathEntry[] paths = new ObservedPathEntry[thePaths.Length];
for (int i = 0; i < thePaths.Length; i++)
{
AbsolutePath absPath = AbsolutePath.Create(pathTable, thePaths[i]);
paths[i] = new ObservedPathEntry(absPath, false, false, false, null, false);
}
var emptyObservedAccessFileNames = SortedReadOnlyArray <StringId, CaseInsensitiveStringIdComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <StringId> .Empty,
new CaseInsensitiveStringIdComparer(pathTable.StringTable));
ObservedPathSet pathSet = new ObservedPathSet(
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <ObservedPathEntry> .FromWithoutCopy(paths),
new ObservedPathEntryExpandedPathComparer(pathTable.ExpandedPathComparer)),
emptyObservedAccessFileNames,
null);
using (var pathSetBuffer = new MemoryStream())
{
using (var writer = new BuildXLWriter(stream: pathSetBuffer, debug: false, leaveOpen: true, logStats: false))
{
pathSet.Serialize(pathTable, writer, preserveCasing: false);
}
pathSetBuffer.Seek(0, SeekOrigin.Begin);
// Must await such that the dispose of the MemoryStream is only after the write completes
return(await session.AddToCasAsync(pathSetBuffer).SuccessAsync());
}
}
/// <summary>
/// Reads a SortedReadOnlyArray
/// </summary>
public SortedReadOnlyArray <TValue, TComparer> ReadSortedReadOnlyArray <TValue, TComparer>(
Func <BuildXLReader, TValue> reader,
TComparer comparer)
where TComparer : class, IComparer <TValue>
{
Contract.Requires(reader != null);
Contract.Requires(comparer != null);
Start <SortedReadOnlyArray <TValue, TComparer> >();
ReadOnlyArray <TValue> array = ReadReadOnlyArray(reader);
End();
return(SortedReadOnlyArray <TValue, TComparer> .FromSortedArrayUnsafe(array, comparer));
}
private static ObservedInputProcessingResult CreateResult(PathTable pathTable, params ObservedInput[] inputs)
{
var sorted = SortedReadOnlyArray <ObservedInput, ObservedInputExpandedPathComparer> .SortUnsafe(
inputs,
new ObservedInputExpandedPathComparer(pathTable.ExpandedPathComparer));
var emptyObservedAccessFileNames = SortedReadOnlyArray <StringId, CaseInsensitiveStringIdComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <StringId> .Empty,
new CaseInsensitiveStringIdComparer(pathTable.StringTable));
return(ObservedInputProcessingResult.CreateForSuccess(
sorted,
emptyObservedAccessFileNames,
dynamicObservations: ReadOnlyArray <(AbsolutePath, DynamicObservationKind)> .Empty,
allowedUndeclaredSourceReads: CollectionUtilities.EmptySet <AbsolutePath>()));
}
private ProcessStrongFingerprintComputationData Convert(ProcessStrongFingerprintComputationData computation)
{
if (AreGraphsSame)
{
return(computation);
}
var pathSet = new ObservedPathSet(
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> .FromSortedArrayUnsafe(
Convert(computation.PathEntries, this, (i, me) => me.Convert(i)),
new ObservedPathEntryExpandedPathComparer(OldModel.PathTable.ExpandedPathComparer)),
computation.PathSet.ObservedAccessedFileNames,
computation.PathSet.UnsafeOptions);
return(computation.UnsafeOverride(
pathSet,
Convert(computation.ObservedInputs, this, (i, me) => me.Convert(i))));
}
public static ObservedPathSet CreatePathSet(PathTable pathTable, params AbsolutePath[] paths)
{
ObservedPathEntry[] entries = paths.Select(p => new ObservedPathEntry(p, false, false, false, null, false)).ToArray();
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> sortedPathIds =
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> .SortUnsafe(
entries,
new ObservedPathEntryExpandedPathComparer(pathTable.ExpandedPathComparer));
var emptyObservedAccessFileNames = SortedReadOnlyArray <StringId, CaseInsensitiveStringIdComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <StringId> .Empty,
new CaseInsensitiveStringIdComparer(pathTable.StringTable));
return(new ObservedPathSet(
sortedPathIds,
emptyObservedAccessFileNames,
new UnsafeOptions(UnsafeOptions.SafeConfigurationValues, new PreserveOutputsInfo(ContentHashingUtilities.CreateRandom(), 0))));
}
/// <summary>
/// Extracts a file into a folder with in manifest based incrementality.
/// </summary>
internal async Task <EvaluationResult> PerformExtractOrIncrementalCheckAsync(DownloadData downloadData)
{
if (m_context.CancellationToken.IsCancellationRequested)
{
return(EvaluationResult.Canceled);
}
// Ensure file is downloaded
var extractedFileResult = await DownloadFile(downloadData);
if (extractedFileResult.IsErrorValue)
{
return(extractedFileResult);
}
var extractedFile = (FileArtifact)extractedFileResult.Value;
var moduleDescriptor = m_workspaceResolver.GetModuleDescriptor(downloadData);
var pipConstructionHelper = PipConstructionHelper.Create(
m_context,
m_frontEndHost.Engine.Layout.ObjectDirectory,
m_frontEndHost.Engine.Layout.RedirectedDirectory,
m_frontEndHost.Engine.Layout.TempDirectory,
m_frontEndHost.PipGraph,
moduleDescriptor.Id,
moduleDescriptor.Name,
RelativePath.Create(downloadData.ModuleSpecFile.GetName(m_context.PathTable)),
FullSymbol.Create(m_context.SymbolTable, "extracted"),
new LocationData(downloadData.ModuleSpecFile, 0, 0),
m_context.QualifierTable.EmptyQualifierId);
// When we don't have to extract we'll expose the downloaded file in the contents.
if (downloadData.Settings.ArchiveType == DownloadArchiveType.File)
{
return(SealDirectory(
pipConstructionHelper,
downloadData,
DirectoryArtifact.CreateWithZeroPartialSealId(downloadData.DownloadedFilePath.GetParent(m_context.PathTable)),
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <FileArtifact> .FromWithoutCopy(new[] { extractedFile }),
OrdinalFileArtifactComparer.Instance)));
}
Statistics.Extractions.Total.Increment();
using (Statistics.Extractions.UpToDateCheckDuration.Start(downloadData.Settings.Url))
{
var result = await CheckIfExtractIsNeededAsync(pipConstructionHelper, downloadData);
if (result.IsErrorValue)
{
Statistics.Extractions.Failures.Increment();
}
if (result != EvaluationResult.Continue)
{
Statistics.Extractions.SkippedDueToManifest.Increment();
return(result);
}
}
using (Statistics.Extractions.Duration.Start(downloadData.Settings.Url))
{
try
{
if (!await Task.Run(
() => TryExtractToDisk(downloadData),
m_context.CancellationToken))
{
Statistics.Extractions.Failures.Increment();
return(EvaluationResult.Error);
}
}
catch (TaskCanceledException)
{
return(EvaluationResult.Canceled);
}
}
using (Statistics.Extractions.UpToDateCheckDuration.Start(downloadData.Settings.Url))
{
var result = await ValidateAndStoreIncrementalExtractState(pipConstructionHelper, downloadData);
if (result.IsErrorValue)
{
Statistics.Extractions.Failures.Increment();
}
return(result);
}
}
/// <nodoc />
public static Possible <ObservedPathSet, DeserializeFailure> TryDeserialize(
PathTable pathTable,
BuildXLReader reader,
PathExpander pathExpander = null,
Func <BuildXLReader, AbsolutePath> pathReader = null,
Func <BuildXLReader, StringId> stringReader = null)
{
PathTable.ExpandedAbsolutePathComparer comparer = pathTable.ExpandedPathComparer;
int pathCount = reader.ReadInt32Compact();
ObservedPathEntry[] paths = new ObservedPathEntry[pathCount];
string lastStr = null;
AbsolutePath lastPath = default(AbsolutePath);
for (int i = 0; i < pathCount; i++)
{
var flags = (ObservedPathEntryFlags)reader.ReadByte();
string enumeratePatternRegex = null;
if ((flags & ObservedPathEntryFlags.DirectoryEnumerationWithCustomPattern) != 0)
{
enumeratePatternRegex = reader.ReadString();
}
else if ((flags & ObservedPathEntryFlags.DirectoryEnumerationWithAllPattern) != 0)
{
enumeratePatternRegex = RegexDirectoryMembershipFilter.AllowAllRegex;
}
AbsolutePath newPath;
string full = null;
if (pathReader != null)
{
newPath = pathReader(reader);
}
else
{
int reuseCount = reader.ReadInt32Compact();
if (reuseCount == 0)
{
full = reader.ReadString();
}
else
{
if (lastStr == null || lastStr.Length < reuseCount)
{
// This path set is invalid.
return(new DeserializeFailure($"Invalid reuseCount: {reuseCount}; last: '{lastStr}', last string length: {lastStr?.Length}"));
}
string partial = reader.ReadString();
full = lastStr.Substring(0, reuseCount) + partial;
}
if (!AbsolutePath.TryCreate(pathTable, full, out newPath))
{
// It might be failed due to the tokenized path.
if (pathExpander == null || !pathExpander.TryCreatePath(pathTable, full, out newPath))
{
return(new DeserializeFailure($"Invalid path: '{full}'"));
}
}
}
paths[i] = new ObservedPathEntry(newPath, flags, enumeratePatternRegex);
if (lastPath.IsValid)
{
#if DEBUG
if (comparer.Compare(lastPath, newPath) >= 0)
{
return(new DeserializeFailure($"Paths not sorted: " +
$"old = '{lastPath.ToString(pathTable)}', new = '{newPath.ToString(pathTable)}';" +
$"old str = '{lastStr}', new str = '{full}'"));
}
#endif
}
lastPath = newPath;
lastStr = full;
}
int fileNameCount = reader.ReadInt32Compact();
StringId[] fileNames = new StringId[fileNameCount];
for (int i = 0; i < fileNameCount; i++)
{
fileNames[i] = stringReader?.Invoke(reader) ?? StringId.Create(pathTable.StringTable, reader.ReadString());
}
// Read unsafe options
var unsafeOptions = UnsafeOptions.TryDeserialize(reader);
if (unsafeOptions == null)
{
return(new DeserializeFailure("UnsafeOptions are null"));
}
// Note that we validated sort order above.
return(new ObservedPathSet(
SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <ObservedPathEntry> .FromWithoutCopy(paths),
new ObservedPathEntryExpandedPathComparer(comparer)),
SortedReadOnlyArray <StringId, CaseInsensitiveStringIdComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <StringId> .FromWithoutCopy(fileNames),
new CaseInsensitiveStringIdComparer(pathTable.StringTable)),
unsafeOptions));
}