/// <inheritdoc />
public Task <Possible <Unit, Failure> > TryMaterializeAsync(
FileRealizationMode fileRealizationModes,
ExpandedAbsolutePath path,
ContentHash contentHash)
{
return(Task.Run <Possible <Unit, Failure> >(
() =>
{
lock (m_lock)
{
CacheEntry entry;
if (m_content.TryGetValue(contentHash, out entry))
{
if ((entry.Sites & CacheSites.Local) == 0)
{
return new Failure <string>("Content is available in 'remote' cache but is not local. Load it locally first with TryLoadAvailableContentAsync.");
}
string expandedPath = path.ExpandedPath;
// IArtifactContentCache prescribes that materialization always produces a 'new' file.
var mayBeDelete = FileUtilities.TryDeletePathIfExists(expandedPath);
if (!mayBeDelete.Succeeded)
{
return mayBeDelete.Failure;
}
try
{
if (m_pathRealizationModes != null)
{
m_pathRealizationModes[expandedPath] = fileRealizationModes;
}
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
FileUtilities.CreateDirectory(Path.GetDirectoryName(expandedPath));
File.WriteAllBytes(expandedPath, entry.Content);
},
ex => { throw new BuildXLException("Failed to materialize content (content found, but couldn't write it)", ex); });
return Unit.Void;
}
catch (BuildXLException ex)
{
return new RecoverableExceptionFailure(ex);
}
}
else
{
return new Failure <string>("Content not found (locally or remotely). Store it first with TryStoreAsync.");
}
}
}));
}
private bool TryWriteSandboxedProcessResult(PathTable pathTable, SandboxedProcessResult result)
{
Contract.Requires(result != null);
// When BuildXL serializes SandboxedProcessInfo, it does not serialize the path table used by SandboxedProcessInfo.
// On deserializing that info, a new path table is created; see the Deserialize method of SandboxedProcessInfo.
// Unix sandbox uses the new path table in the deserialized SandboxedProcessInfo to create ManifestPath (AbsolutePath)
// from reported path access (string) in ReportFileAccess. Without special case, the serialization of SandboxedProcessResult
// will serialize the AbsolutePath as is. Then, when SandboxedProcessResult is read by BuildXL, BuildXL will not understand
// the ManifestPath because it is created from a different path table.
//
// In Windows, instead of creating ManifestPath from the reported path access (string), ManifestPath is reported from Detours
// using the AbsolutePath id embedded in the file access manifest. That AbsolutePath id is obtained using the same
// path table used by BuildXL, and thus BuildXL will understand the ManifestPath serialized by this tool.
//
// For Unix, we need to give a special care of path serialization.
bool isWindows = !OperatingSystemHelper.IsUnixOS;
Action <BuildXLWriter, AbsolutePath> writePath = (writer, path) =>
{
if (isWindows)
{
writer.Write(true);
writer.Write(path);
}
else
{
writer.Write(false);
writer.Write(path.ToString(pathTable));
}
};
bool success = false;
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
string sandboxedProcessResultOutputPath = Path.GetFullPath(m_configuration.SandboxedProcessResultOutputFile);
m_logger.LogInfo($"Writing sandboxed process result to '{sandboxedProcessResultOutputPath}'");
using (FileStream stream = File.OpenWrite(sandboxedProcessResultOutputPath))
{
result.Serialize(stream, writePath);
}
success = true;
},
ex =>
{
m_logger.LogError(ex.ToString());
success = false;
});
return(success);
}
private void Save(FileEnvelopeId atomicSaveToken, string path)
{
Contract.Requires(!string.IsNullOrWhiteSpace(path));
FileUtilities.DeleteFile(path);
FileUtilities.CreateDirectory(Path.GetDirectoryName(path));
using (var stream = FileUtilities.CreateFileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.Delete,
// Do not write the file with SequentialScan since it will be reread in the subsequent build
FileOptions.None))
{
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
using (var pm = BuildXL.Tracing.PerformanceMeasurement.StartWithoutStatistic(
m_loggingContext,
loggingContext => Logger.Log.StartSavingChangeTracker(loggingContext, path),
loggingContext => Logger.Log.EndSavingChangeTracker(loggingContext)))
{
Stopwatch sw = Stopwatch.StartNew();
FileEnvelope.WriteHeader(stream, atomicSaveToken);
using (var writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false))
{
if (IsDisabledOrNullTrackingSet)
{
writer.Write(true);
}
else
{
writer.Write(false);
writer.Write(m_buildEngineFingerprint, (w, s) => w.Write(s));
m_changeTrackingSet.Save(writer);
}
}
FileEnvelope.FixUpHeader(stream, atomicSaveToken);
Logger.Log.SavingChangeTracker(
pm.LoggingContext,
path,
atomicSaveToken.ToString(),
m_changeTrackingSet == null ? "Null" : TrackingState.ToString(),
m_changeTrackingSet == null ? 0 : m_changeTrackingSet.TrackedVolumes.Count(),
sw.ElapsedMilliseconds);
}
},
ex => { throw new BuildXLException("Failed to save file change tracker", ex); });
}
}
public override async Task <SandboxedProcessResult> GetResultAsync()
{
Contract.Requires(m_processExecutor != null);
// (1) Wait for VmCommandProxy.
await m_processExecutor.WaitForExitAsync();
await m_processExecutor.WaitForStdOutAndStdErrAsync();
// (2) Validate result of VmCommandProxy.
if (m_processExecutor.TimedOut || m_processExecutor.Killed)
{
// If timed out/killed, then sandboxed process result may have not been deserialized yet.
return(CreateResultForVmCommandProxyFailure());
}
if (Process.ExitCode != 0)
{
return(CreateResultForVmCommandProxyFailure());
}
if (!FileUtilities.FileExistsNoFollow(RunOutputPath))
{
m_error.AppendLine($"Could not find VM output file '{RunOutputPath}");
return(CreateResultForVmCommandProxyFailure());
}
try
{
// (3) Validate the result of sandboxed process executor run by VmCommandProxy.
RunResult runVmResult = ExceptionUtilities.HandleRecoverableIOException(
() => VmSerializer.DeserializeFromFile <RunResult>(RunOutputPath),
e => m_error.AppendLine(e.Message));
if (runVmResult == null)
{
return(CreateResultForVmCommandProxyFailure());
}
if (runVmResult.ProcessStateInfo.ExitCode != 0)
{
return(CreateResultForSandboxExecutorFailure(runVmResult));
}
}
catch (Exception e)
{
m_error.AppendLine(e.ToString());
return(CreateResultForVmCommandProxyFailure());
}
return(DeserializeSandboxedProcessResultFromFile());
}
private Task <Possible <ContentHash, Failure> > TryStoreInternalAsync(
ExpandedAbsolutePath path,
FileRealizationMode fileRealizationModes,
ContentHash?knownContentHash)
{
return(Task.Run <Possible <ContentHash, Failure> >(
() =>
{
lock (m_lock)
{
byte[] contentBytes = ExceptionUtilities.HandleRecoverableIOException(
() => { return File.ReadAllBytes(path.ExpandedPath); },
ex => { throw new BuildXLException("Failed to store content (couldn't read new content from disk)", ex); });
ContentHash contentHash = ContentHashingUtilities.HashBytes(contentBytes);
if (knownContentHash.HasValue && contentHash != knownContentHash.Value)
{
return new Failure <string>(I($"Stored content had an unexpected hash. (expected: {knownContentHash.Value}; actual: {contentHash})"));
}
CacheEntry entry;
if (m_content.TryGetValue(contentHash, out entry))
{
// We assume that stores of content already present somewhere still cause replication
// to both the local and remote sites. See class remarks.
entry.Sites |= CacheSites.LocalAndRemote;
return contentHash;
}
else
{
try
{
if (m_pathRealizationModes != null)
{
m_pathRealizationModes[path.ExpandedPath] = fileRealizationModes;
}
// We assume that stored content is instantly and magically replicated to some remote place.
// See class remarks.
m_content[contentHash] = new CacheEntry(contentBytes, CacheSites.LocalAndRemote);
return contentHash;
}
catch (BuildXLException ex)
{
return new RecoverableExceptionFailure(ex);
}
}
}
}));
}
private bool CreateAndPreventDeletionHelper(string path)
{
Contract.Requires(!string.IsNullOrEmpty(path));
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
Directory.CreateDirectory(path);
},
ex => { throw new BuildXLException(I($"Failed to create the directory '{path}'"), ex); });
return(LockDirectory(path, false));
}
private static Possible <string> TryReadCacheConfigFile(string path)
{
try
{
return(ExceptionUtilities.HandleRecoverableIOException(
() => File.ReadAllText(path),
ex => { throw new BuildXLException("Unable to read cache configuration", ex); }));
}
catch (BuildXLException ex)
{
return(new RecoverableExceptionFailure(ex));
}
}
private byte[] CalculateBytes(Stream content)
{
byte[] contentBytes = ExceptionUtilities.HandleRecoverableIOException(
() =>
{
MemoryStream memoryStream;
Stream streamToRead;
if (!content.CanSeek)
{
memoryStream = new MemoryStream();
streamToRead = memoryStream;
}
else
{
memoryStream = null;
streamToRead = content;
}
using (memoryStream)
{
if (memoryStream != null)
{
content.CopyTo(memoryStream);
memoryStream.Position = 0;
}
Contract.Assert(streamToRead.CanSeek);
Contract.Assume(streamToRead.Length <= int.MaxValue);
var length = (int)streamToRead.Length;
var contentBytesLocal = new byte[length];
int read = 0;
while (read < length)
{
int readThisIteration = streamToRead.Read(contentBytesLocal, read, length - read);
if (readThisIteration == 0)
{
throw new BuildXLException("Unexpected end of stream");
}
read += readThisIteration;
}
return(contentBytesLocal);
}
},
ex => throw new BuildXLException("Undable to calculate bytes from stream"));
return(contentBytes);
}
/// <inheritdoc />
public Task <Possible <ContentAvailabilityBatchResult, Failure> > TryLoadAvailableContentAsync(IReadOnlyList <ContentHash> hashes)
{
return(Task.Run <Possible <ContentAvailabilityBatchResult, Failure> >(
() =>
{
lock (m_lock)
{
bool allAvailable = true;
var results = new ContentAvailabilityResult[hashes.Count];
for (int i = 0; i < hashes.Count; i++)
{
CacheEntry entry;
bool available = m_content.TryGetValue(hashes[i], out entry) && entry.Sites != CacheSites.None;
long bytesTransferredRemotely = 0;
if (available)
{
if ((entry.Sites & CacheSites.Local) == 0)
{
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
// Copy to local side.
bytesTransferredRemotely = Download(hashes[i]);
},
ex => throw new BuildXLException(I($"Failed to load content '{hashes[i]}' from remote site"), ex));
}
entry.Sites |= CacheSites.Local;
}
results[i] = new ContentAvailabilityResult(
hashes[i],
isAvailable: available,
bytesTransferred: bytesTransferredRemotely,
sourceCache: nameof(MockArtifactContentCache));
allAvailable &= available;
}
return new ContentAvailabilityBatchResult(
ReadOnlyArray <ContentAvailabilityResult> .FromWithoutCopy(results),
allContentAvailable: allAvailable);
}
}));
}
private void HandleRecoverableIOException(Action action)
{
try
{
ExceptionUtilities.HandleRecoverableIOException(
action,
ex => { throw new BuildXLException("Writing file failed", ex); });
}
catch (BuildXLException ex)
{
m_exception = ex;
ReleaseTextWriter();
ReleaseStringBuilder();
m_fileName = null;
m_length = SandboxedProcessOutput.NoLength;
IsFrozen = true;
}
}
private void ProcessQueueItem(Pip pip, MutablePipState pipState)
{
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
var start = Stopwatch.GetTimestamp();
PageableStoreId value = m_store.Write(writer => ((PipWriter)writer).Write(pip));
#if DEBUG
m_store.Read <Pip>(value, reader => ((PipReader)reader).ReadPip());
#endif
pipState.StoreId = value;
GC.KeepAlive(pip); // Pip must not get GCed until after StoreId has been set.
var end = Stopwatch.GetTimestamp();
Interlocked.Add(ref m_writeTicks, end - start);
Interlocked.Increment(ref m_writes);
},
ex => ExceptionHandling.OnFatalException(ex));
}
private bool TryReadSandboxedProcessInfo(out SandboxedProcessInfo sandboxedProcessInfo)
{
sandboxedProcessInfo = null;
sandboxedProcessInfo = ExceptionUtilities.HandleRecoverableIOException(
() =>
{
using (FileStream stream = File.OpenRead(Path.GetFullPath(m_configuration.SandboxedProcessInfoInputFile)))
{
// TODO: Custom DetoursEventListener?
return(SandboxedProcessInfo.Deserialize(stream, m_loggingContext, detoursEventListener: null));
}
},
ex =>
{
m_logger.LogError(ex.ToString());
});
return(sandboxedProcessInfo != null);
}
/// <inheritdoc />
public Task MoveFileAsync(
string source,
string destination,
bool replaceExisting = false)
{
Contract.Requires(!string.IsNullOrEmpty(source));
Contract.Requires(!string.IsNullOrEmpty(destination));
return(Task.Run(
() => ExceptionUtilities.HandleRecoverableIOException(
() =>
{
if (replaceExisting)
{
DeleteFile(destination);
}
File.Move(source, destination);
},
ex => { throw new BuildXLException(I($"File move from '{source}' to '{destination}' failed"), ex); })));
}
private void EnsureRemote(ContentHash hash)
{
bool entryExists = m_content.TryGetValue(hash, out CacheEntry entry);
Contract.Assert(entryExists);
Contract.Assert(entry.Sites != CacheSites.None);
GetPaths(hash, out string localPath, out string remotePath);
if ((entry.Sites & CacheSites.Remote) == 0)
{
Contract.Requires((entry.Sites & CacheSites.Local) != 0);
ExceptionUtilities.HandleRecoverableIOException(
() => Upload(hash),
ex => throw new BuildXLException(I($"Unable to ensure remote replication of content hash '{hash.ToString()}'")));
}
else
{
Contract.Assert(File.Exists(remotePath));
}
entry.Sites |= CacheSites.Remote;
}
private static void SaveEntries(PathTable pathTable, AbsolutePath destination, List <Entry> entries)
{
var destinationPath = destination.ToString(pathTable);
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
// Build the report content. Materializes to the file on a per line basis since the whole content may be very big.
using (var stream = new StreamWriter(destinationPath))
{
stream.WriteLine(GetHeader());
foreach (var entry in entries)
{
stream.WriteLine(GetLine(entry));
}
}
},
ex =>
{
throw new BuildXLException("Error while producing the report. Inner exception reason: " + ex.Message, ex);
});
}
/// <summary>
/// Reads the entire value; don't call this if the length is excessive, as it might OOM.
/// </summary>
/// <exception cref="BuildXLException">Thrown if a recoverable error occurs while opening the stream.</exception>
public async Task <string> ReadValueAsync()
{
if (m_exception != null)
{
ExceptionDispatchInfo.Capture(m_exception).Throw();
}
if (m_value != null)
{
return(m_value);
}
return(await ExceptionUtilities.HandleRecoverableIOException(
async() =>
{
using (TextReader reader = CreateFileReader())
{
string value = await reader.ReadToEndAsync();
return value;
}
},
e => throw new BuildXLException("Failed to read a value from a stream", e)));
}
/// <summary>
/// Returns a new <see cref="FileStream" /> with the given creation mode, access level, and sharing.
/// </summary>
/// <remarks>
/// This factory exists purely because FileStream cannot be constructed in async mode without also specifying a buffer
/// size.
/// We go ahead and wrap recoverable errors as BuildXLExceptions as well.
///
/// Does not support paths longer than MAX_PATH
/// </remarks>
/// <exception cref="BuildXLException">
/// Thrown if a recoverable error occurs while opening the stream (see
/// <see cref="ExceptionUtilities.HandleRecoverableIOException{T}" />)
/// </exception>
/// <param name="path">Path of file</param>
/// <param name="fileMode">FileMode</param>
/// <param name="fileAccess">FileAccess</param>
/// <param name="fileShare">FileShare</param>
/// <param name="options">FileOptions</param>
/// <param name="force">Clears the readonly attribute if necessary</param>
/// <param name="allowExcludeFileShareDelete">Indicates willful omission of FILE_SHARE_DELETE.</param>
public static FileStream CreateFileStream(
string path,
FileMode fileMode,
FileAccess fileAccess,
FileShare fileShare,
FileOptions options = FileOptions.None,
bool force = false,
bool allowExcludeFileShareDelete = false)
{
Contract.Requires(allowExcludeFileShareDelete || ((fileShare & FileShare.Delete) != 0));
Contract.EnsuresOnThrow <BuildXLException>(true);
// The bufferSize of 4096 bytes is the default as used by the other FileStream constructors
// http://index/mscorlib/system/io/filestream.cs.html
return(ExceptionUtilities.HandleRecoverableIOException(
() =>
{
try
{
return new FileStream(path, fileMode, fileAccess, fileShare, bufferSize: DefaultBufferSize, options: options);
}
catch (UnauthorizedAccessException)
{
// This is a workaround to allow write access to a file that is marked as readonly. It is
// exercised when hashing the output files of pips that create readonly files. The hashing currently
// opens files as write
if (force)
{
File.SetAttributes(path, File.GetAttributes(path) & ~FileAttributes.ReadOnly);
return new FileStream(path, fileMode, fileAccess, fileShare, bufferSize: DefaultBufferSize, options: options);
}
throw;
}
},
ex => { throw new BuildXLException(string.Format(CultureInfo.InvariantCulture, "Failed to open path '{0}'", path), ex); }));
}
/// <inheritdoc />
public FileStream CreateFileStream(
string path,
FileMode fileMode,
FileAccess fileAccess,
FileShare fileShare,
FileOptions options,
bool force)
{
// The bufferSize of 4096 bytes is the default as used by the other FileStream constructors
// http://index/mscorlib/system/io/filestream.cs.html
return(ExceptionUtilities.HandleRecoverableIOException(
() =>
{
try
{
return new FileStream(path, fileMode, fileAccess, fileShare, bufferSize: DefaultBufferSize, options: options);
}
catch (UnauthorizedAccessException)
{
// This is a workaround to allow write access to a file that is marked as readonly. It is
// exercised when hashing the output files of pips that create readonly files. The hashing currently
// opens files as write
if (force)
{
File.SetAttributes(path, File.GetAttributes(path) & ~FileAttributes.ReadOnly);
return new FileStream(path, fileMode, fileAccess, fileShare, bufferSize: DefaultBufferSize, options: options);
}
throw;
}
},
ex =>
{
throw new BuildXLException(I($"Failed to open path '{path}' with mode='{fileMode}', access='{fileAccess}', share='{fileShare}'"), ex);
}));
}
private bool TryWriteSandboxedProcessResult(SandboxedProcessResult result)
{
Contract.Requires(result != null);
bool success = false;
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
using (FileStream stream = File.OpenWrite(Path.GetFullPath(m_configuration.SandboxedProcessResultOutputFile)))
{
result.Serialize(stream);
}
success = true;
},
ex =>
{
m_logger.LogError(ex.ToString());
success = false;
});
return(success);
}
private void SaveInternal(string fileContentTablePath)
{
Contract.Requires(!string.IsNullOrWhiteSpace(fileContentTablePath));
Contract.EnsuresOnThrow <BuildXLException>(true);
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
int numEvicted = 0;
Directory.CreateDirectory(Path.GetDirectoryName(fileContentTablePath));
// Note that we are using a non-async file stream here. That's because we're doing lots of tiny writes for simplicity,
// but tiny writes on an async stream end up blocking anyway while adding silly overhead.
using (FileStream stream = FileUtilities.CreateFileStream(
fileContentTablePath,
FileMode.Create,
FileAccess.Write,
FileShare.Delete,
// Do not write the file with SequentialScan since it will be reread in the subsequent build
FileOptions.None))
{
// We don't have anything in particular to correlate this file to,
// so we are simply creating a unique correlation id that is used as part
// of the header consistency check.
FileEnvelopeId correlationId = FileEnvelopeId.Create();
s_fileEnvelope.WriteHeader(stream, correlationId);
using (var writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false))
{
long numberOfEntriesPosition = writer.BaseStream.Position;
writer.Write(0U);
uint entriesWritten = 0;
var hashBuffer = new byte[ContentHashingUtilities.HashInfo.ByteLength];
foreach (var fileAndEntryPair in m_entries)
{
// Skip saving anything with a TTL of zero. These entries were loaded
// with a TTL of one (immediately decremented) and were not used since load.
// See class remarks.
if (fileAndEntryPair.Value.TimeToLive == 0)
{
numEvicted++;
continue;
}
// Key: Volume and File ID
fileAndEntryPair.Key.Serialize(writer);
// Entry: USN, hash, time to live.
writer.Write(fileAndEntryPair.Value.Usn.Value);
fileAndEntryPair.Value.Hash.SerializeHashBytes(hashBuffer, 0);
writer.Write(hashBuffer);
writer.Write(fileAndEntryPair.Value.Length);
writer.Write(fileAndEntryPair.Value.TimeToLive);
entriesWritten++;
}
var endPosition = writer.BaseStream.Position;
writer.BaseStream.Position = numberOfEntriesPosition;
writer.Write(entriesWritten);
writer.BaseStream.Position = endPosition;
}
s_fileEnvelope.FixUpHeader(stream, correlationId);
}
Counters.AddToCounter(FileContentTableCounters.NumEvicted, numEvicted);
return(Unit.Void);
},
ex => { throw new BuildXLException("Failure writing file content table", ex); });
}
/// <summary>
/// Attempts to create auto-complete items from the "tagged template" statements for file, path, directory and relative path (f``, d``, p``)
/// </summary>
internal static IEnumerable <CompletionItem> TryCreateFileCompletionItemsForTaggedTemplatedExpression(CompletionState completionState, INode completionNode)
{
var templateExpression = completionNode.Cast <ITaggedTemplateExpression>();
// GetTemplateText does a Cast on the expression and then access the "text" property
// instead of just returning a null or empty string.
// So, before we can use GetTemplateText, we must check to see if it is actually of
// the correct type.
// We cannot use the syntax kind, as ILiteralExpression can cover many syntax kinds -
// Strings, numbers, etc.
var literalExprssion = templateExpression?.TemplateExpression?.As <ILiteralExpression>();
var existingFileExpressionText = literalExprssion?.GetTemplateText();
if (!string.IsNullOrEmpty(existingFileExpressionText))
{
if (!RelativePath.TryCreate(completionState.PathTable.StringTable, existingFileExpressionText, out var normalizedPath))
{
return(null);
}
existingFileExpressionText = normalizedPath.ToString(completionState.PathTable.StringTable);
}
else
{
existingFileExpressionText = string.Empty;
}
var sourceFileRootPath = Path.GetDirectoryName(completionState.StartingNode.SourceFile.ToUri().LocalPath);
// Now, for this bit of fun. If the user has started typing a path already, then we still
// want to complete what they have started typing.
// If what they have typed does not have a directory separator in it, then we will use
// that as the search pattern below. If it does however, then we will combine that
// with the spec directory and then attempt to get the remainder to use as the search path.
// So... if the user has typed `foo` we will use the spec path as the root search directory
// and `foo*` as the search pattern.
// If the user has typed `foo\bar` then we will use `<spec path>\foo` as the search path and
// "bar" as the search pattern.
var originalSourceFileRootPath = sourceFileRootPath;
if (existingFileExpressionText.Contains(Path.DirectorySeparatorChar))
{
sourceFileRootPath = Path.Combine(sourceFileRootPath, existingFileExpressionText);
existingFileExpressionText = Path.GetFileName(sourceFileRootPath);
sourceFileRootPath = Path.GetDirectoryName(sourceFileRootPath);
}
// If the user types just a "\" character, then the source file root path can be null
if (string.IsNullOrEmpty(sourceFileRootPath))
{
return(null);
}
// If we managed to get a path that is outside of our spec path, then bail. Do not allow auto completion
if (!sourceFileRootPath.StartsWith(originalSourceFileRootPath, System.StringComparison.OrdinalIgnoreCase) ||
!Directory.Exists(sourceFileRootPath))
{
return(null);
}
bool isDirectoryTag = templateExpression.GetInterpolationKind() == InterpolationKind.DirectoryInterpolation;
// Leverage BuildXL's recoverable IO exception extension to ensure that if we hit
// cases like "access denied" that it does not take down the plugin.
IEnumerable <string> fileSystemEntries = null;
try
{
fileSystemEntries = ExceptionUtilities.HandleRecoverableIOException(
() =>
{
return(isDirectoryTag ?
Directory.EnumerateDirectories(sourceFileRootPath, existingFileExpressionText + "*", SearchOption.TopDirectoryOnly) :
Directory.EnumerateFileSystemEntries(sourceFileRootPath, existingFileExpressionText + "*", SearchOption.TopDirectoryOnly));
},
ex =>
{
throw new BuildXLException(ex.Message);
});
}
catch (BuildXLException)
{
}
if (fileSystemEntries.IsNullOrEmpty())
{
return(null);
}
// We aren't done yet :)
// Let's see if we can filter further if we are in an array literal experssion..
// So, if the user has something like this
// sources: [
// f`fileA.cpp`,
// f`fileB.cpp`,
//
// And they have files fileC-FileZ.cpp on disk, we filter out fileA.cpp and fileB.cpp.
// This is very similar to filtering out properties that are already set on an object
// literal.
if (templateExpression.Parent?.Kind == SyntaxKind.ArrayLiteralExpression)
{
var existingFiles = new List <string>();
var existingArrayLiteralExpression = templateExpression.Parent.Cast <IArrayLiteralExpression>();
foreach (var existingFileItem in existingArrayLiteralExpression.Elements)
{
if (existingFileItem.Kind == SyntaxKind.TaggedTemplateExpression)
{
var existingFileTemplateExpression = existingFileItem.Cast <ITaggedTemplateExpression>();
// GetTemplateText does a Cast on the expression and then access the "text" property
// instead of just returning a null or empty string.
// So, before we can use GetTemplateText, we must check to see if it is actually of
// the correct type.
// We cannot use the syntax kind, as ILiteralExpression can cover many syntax kinds -
// Strings, numbers, etc.
var existingFileLiteralExprssion = existingFileTemplateExpression?.TemplateExpression?.As <ILiteralExpression>();
var existingFilePathText = existingFileLiteralExprssion?.GetTemplateText();
if (!string.IsNullOrEmpty(existingFilePathText))
{
existingFiles.Add(Path.Combine(originalSourceFileRootPath, existingFilePathText));
}
}
}
fileSystemEntries = fileSystemEntries.Where(possibleEntry => !existingFiles.Contains(possibleEntry, StringComparer.InvariantCultureIgnoreCase));
}
return(fileSystemEntries.Select(name =>
{
var itemLabel = name.Substring(sourceFileRootPath.Length + 1);
var item = new CompletionItem()
{
Kind = CompletionItemKind.File,
Label = itemLabel,
};
return item;
}));
}
private Task <Possible <ContentHash, Failure> > TryStoreInternalAsync(
ExpandedAbsolutePath path,
FileRealizationMode fileRealizationModes,
ContentHash?knownContentHash)
{
return(Task.Run <Possible <ContentHash, Failure> >(
() =>
{
lock (m_lock)
{
byte[] contentBytes = ExceptionUtilities.HandleRecoverableIOException(
() =>
{
var expandedPath = path.ExpandedPath;
using (FileStream fileStream = new FileStream(expandedPath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (BinaryReader binaryReader = new BinaryReader(fileStream))
{
// This will work with files up to 2GB in length, due to the 'int' API signature
return binaryReader.ReadBytes((int)new FileInfo(expandedPath).Length);
}
},
ex => { throw new BuildXLException("Failed to store content (couldn't read new content from disk)", ex); });
ContentHash contentHash = ContentHashingUtilities.HashBytes(contentBytes);
if (knownContentHash.HasValue && contentHash != knownContentHash.Value)
{
return new Failure <string>(I($"Stored content had an unexpected hash. (expected: {knownContentHash.Value}; actual: {contentHash})"));
}
CacheEntry entry;
if (m_content.TryGetValue(contentHash, out entry))
{
// We assume that stores of content already present somewhere still cause replication
// to both the local and remote sites. See class remarks.
entry.Sites |= CacheSites.LocalAndRemote;
return contentHash;
}
else
{
try
{
if (m_pathRealizationModes != null)
{
m_pathRealizationModes[path.ExpandedPath] = fileRealizationModes;
}
// We assume that stored content is instantly and magically replicated to some remote place.
// See class remarks.
m_content[contentHash] = new CacheEntry(contentBytes, CacheSites.LocalAndRemote);
return contentHash;
}
catch (BuildXLException ex)
{
return new RecoverableExceptionFailure(ex);
}
}
}
}));
}
public static LoadingTrackerResult TryLoad(
LoggingContext loggingContext,
string path,
VolumeMap volumeMap,
IChangeJournalAccessor journal,
string buildEngineFingerprint,
bool loadForAllCapableVolumes = true)
{
Contract.Requires(loggingContext != null);
Contract.Requires(!loadForAllCapableVolumes || volumeMap != null);
Contract.Requires(!loadForAllCapableVolumes || journal != null);
Stopwatch stopwatch = Stopwatch.StartNew();
SafeFileHandle handle;
OpenFileResult result = FileUtilities.TryCreateOrOpenFile(
path,
FileDesiredAccess.GenericRead,
FileShare.Read | FileShare.Delete,
FileMode.Open,
// Ok to evict the file from standby since the file will be overwritten and never reread from disk after this point.
FileFlagsAndAttributes.FileFlagSequentialScan,
out handle);
if (result.Succeeded)
{
Contract.Assume(handle != null);
Contract.Assume(!handle.IsInvalid);
using (handle)
{
using (var stream = new FileStream(handle, FileAccess.Read))
{
FileEnvelopeId fileEnvelopeId;
try
{
fileEnvelopeId = FileEnvelope.ReadHeader(stream);
}
catch (BuildXLException e)
{
return(LoadingTrackerResult.FailBadFormatMarker(e.Message));
}
try
{
return(ExceptionUtilities.HandleRecoverableIOException(
() =>
{
using (var reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: true))
{
bool wasTrackerDisabled = reader.ReadBoolean();
if (wasTrackerDisabled)
{
return LoadingTrackerResult.FailPriorTrackerDisabled();
}
var previousFingerprint = reader.ReadNullable(r => r.ReadString());
// only check for fingerprints match if the supplied fingerprint is valid
// this is to support special cases where we might want to load ChangeTracker
// regardless of the previously stored fingerprint value
if (buildEngineFingerprint != null && !string.Equals(previousFingerprint, buildEngineFingerprint, StringComparison.Ordinal))
{
return LoadingTrackerResult.FailBuildEngineFingerprintMismatch();
}
return FileChangeTrackingSet.TryLoad(
loggingContext,
fileEnvelopeId,
reader,
volumeMap,
journal,
stopwatch,
loadForAllCapableVolumes);
}
},
ex =>
{
throw new BuildXLException(ex.Message);
}));
}
catch (Exception e)
{
// Catch any exception. Failure in loading FileChangeTracker should not
// cause build break, or worse, make people stuck on erroneous state.
// In such a case, BuildXL simply has to start tracking from scratch.
return(LoadingTrackerResult.FailLoadException(e.GetLogEventMessage()));
}
}
}
}
Contract.Assume(handle == null);
return(LoadingTrackerResult.FailTrackingSetCannotBeOpened(result.CreateFailureForError().DescribeIncludingInnerFailures()));
}
private Task <Possible <ContentHash, Failure> > TryStoreInternalAsync(Stream content, ContentHash?knownContentHash)
{
return(Task.Run <Possible <ContentHash, Failure> >(
() =>
{
lock (m_lock)
{
CacheEntry entry;
if (knownContentHash.HasValue && m_content.TryGetValue(knownContentHash.Value, out entry))
{
// We assume that stores of content already present somewhere still cause replication
// to both the local and remote sites. See class remarks.
entry.Sites |= CacheSites.LocalAndRemote;
return knownContentHash.Value;
}
else
{
try
{
byte[] contentBytes = ExceptionUtilities.HandleRecoverableIOException(
() =>
{
MemoryStream memoryStream;
Stream streamToRead;
if (!content.CanSeek)
{
memoryStream = new MemoryStream();
streamToRead = memoryStream;
}
else
{
memoryStream = null;
streamToRead = content;
}
using (memoryStream)
{
if (memoryStream != null)
{
content.CopyTo(memoryStream);
memoryStream.Position = 0;
}
Contract.Assert(streamToRead.CanSeek);
Contract.Assume(streamToRead.Length <= int.MaxValue);
var length = (int)streamToRead.Length;
var contentBytesLocal = new byte[length];
int read = 0;
while (read < length)
{
int readThisIteration = streamToRead.Read(contentBytesLocal, read, length - read);
if (readThisIteration == 0)
{
throw new BuildXLException("Unexpected end of stream");
}
read += readThisIteration;
}
return contentBytesLocal;
}
},
ex => { throw new BuildXLException("Failed to read content from the provided stream in order to store it", ex); });
ContentHash contentHash = knownContentHash ?? ContentHashingUtilities.HashBytes(contentBytes);
// We assume that stored content is instantly and magically replicated to some remote place.
// See class remarks.
m_content[contentHash] = new CacheEntry(contentBytes, CacheSites.LocalAndRemote);
return contentHash;
}
catch (BuildXLException ex)
{
return new RecoverableExceptionFailure(ex);
}
}
}
}));
}
private Task <Possible <ContentHash, Failure> > TryStoreInternalAsync(
ExpandedAbsolutePath path,
FileRealizationMode fileRealizationMode,
ContentHash?knownContentHash)
{
return(Task.Run <Possible <ContentHash, Failure> >(
() =>
{
lock (m_lock)
{
string expandedPath = path.ExpandedPath;
ContentHash contentHash;
if (knownContentHash.HasValue)
{
contentHash = knownContentHash.Value;
var mayBeHash = TryHashFile(expandedPath);
if (!mayBeHash.Succeeded)
{
return mayBeHash.Failure;
}
if (contentHash != mayBeHash.Result)
{
return new Failure <string>(I($"Stored content had an unexpected hash. (expected: {contentHash}; actual: {mayBeHash.Result})"));
}
}
else
{
var maybeHash = TryHashFile(expandedPath);
if (!maybeHash.Succeeded)
{
return maybeHash.Failure;
}
contentHash = maybeHash.Result;
}
string localPath = GetLocalPath(contentHash);
CacheEntry entry;
if (m_content.TryGetValue(contentHash, out entry))
{
EnsureLocal(contentHash);
EnsureRemote(contentHash);
return ExceptionUtilities.HandleRecoverableIOException <Possible <ContentHash, Failure> >(
() =>
{
var putFile = PutFileInternal(expandedPath, contentHash, fileRealizationMode);
if (!putFile.Succeeded)
{
return putFile.Failure;
}
return contentHash;
},
ex => throw new BuildXLException(I($"Failed to store '{expandedPath}'"), ex));
}
else
{
var result = ExceptionUtilities.HandleRecoverableIOException <Possible <ContentHash, Failure> >(
() =>
{
var putFile = PutFileInternal(expandedPath, contentHash, fileRealizationMode);
if (!putFile.Succeeded)
{
return putFile.Failure;
}
return contentHash;
},
ex => throw new BuildXLException(I($"Failed to store '{expandedPath}'"), ex));
m_content.Add(contentHash, new CacheEntry(CacheSites.Local));
EnsureRemote(contentHash);
m_pathRealizationModes[expandedPath] = fileRealizationMode;
return result;
}
}
}));
}
