/// <inheritdoc /> public Task <Possible <Unit, Failure> > TryMaterializeAsync( FileRealizationMode fileRealizationModes, ExpandedAbsolutePath path, ContentHash contentHash) { return(Task.Run <Possible <Unit, Failure> >( () => { lock (m_lock) { CacheEntry entry; if (m_content.TryGetValue(contentHash, out entry)) { if ((entry.Sites & CacheSites.Local) == 0) { return new Failure <string>("Content is available in 'remote' cache but is not local. Load it locally first with TryLoadAvailableContentAsync."); } string expandedPath = path.ExpandedPath; // IArtifactContentCache prescribes that materialization always produces a 'new' file. var mayBeDelete = FileUtilities.TryDeletePathIfExists(expandedPath); if (!mayBeDelete.Succeeded) { return mayBeDelete.Failure; } try { if (m_pathRealizationModes != null) { m_pathRealizationModes[expandedPath] = fileRealizationModes; } ExceptionUtilities.HandleRecoverableIOException( () => { FileUtilities.CreateDirectory(Path.GetDirectoryName(expandedPath)); File.WriteAllBytes(expandedPath, entry.Content); }, ex => { throw new BuildXLException("Failed to materialize content (content found, but couldn't write it)", ex); }); return Unit.Void; } catch (BuildXLException ex) { return new RecoverableExceptionFailure(ex); } } else { return new Failure <string>("Content not found (locally or remotely). Store it first with TryStoreAsync."); } } })); }
private bool TryWriteSandboxedProcessResult(PathTable pathTable, SandboxedProcessResult result) { Contract.Requires(result != null); // When BuildXL serializes SandboxedProcessInfo, it does not serialize the path table used by SandboxedProcessInfo. // On deserializing that info, a new path table is created; see the Deserialize method of SandboxedProcessInfo. // Unix sandbox uses the new path table in the deserialized SandboxedProcessInfo to create ManifestPath (AbsolutePath) // from reported path access (string) in ReportFileAccess. Without special case, the serialization of SandboxedProcessResult // will serialize the AbsolutePath as is. Then, when SandboxedProcessResult is read by BuildXL, BuildXL will not understand // the ManifestPath because it is created from a different path table. // // In Windows, instead of creating ManifestPath from the reported path access (string), ManifestPath is reported from Detours // using the AbsolutePath id embedded in the file access manifest. That AbsolutePath id is obtained using the same // path table used by BuildXL, and thus BuildXL will understand the ManifestPath serialized by this tool. // // For Unix, we need to give a special care of path serialization. bool isWindows = !OperatingSystemHelper.IsUnixOS; Action <BuildXLWriter, AbsolutePath> writePath = (writer, path) => { if (isWindows) { writer.Write(true); writer.Write(path); } else { writer.Write(false); writer.Write(path.ToString(pathTable)); } }; bool success = false; ExceptionUtilities.HandleRecoverableIOException( () => { string sandboxedProcessResultOutputPath = Path.GetFullPath(m_configuration.SandboxedProcessResultOutputFile); m_logger.LogInfo($"Writing sandboxed process result to '{sandboxedProcessResultOutputPath}'"); using (FileStream stream = File.OpenWrite(sandboxedProcessResultOutputPath)) { result.Serialize(stream, writePath); } success = true; }, ex => { m_logger.LogError(ex.ToString()); success = false; }); return(success); }
private void Save(FileEnvelopeId atomicSaveToken, string path) { Contract.Requires(!string.IsNullOrWhiteSpace(path)); FileUtilities.DeleteFile(path); FileUtilities.CreateDirectory(Path.GetDirectoryName(path)); using (var stream = FileUtilities.CreateFileStream( path, FileMode.Create, FileAccess.Write, FileShare.Delete, // Do not write the file with SequentialScan since it will be reread in the subsequent build FileOptions.None)) { ExceptionUtilities.HandleRecoverableIOException( () => { using (var pm = BuildXL.Tracing.PerformanceMeasurement.StartWithoutStatistic( m_loggingContext, loggingContext => Logger.Log.StartSavingChangeTracker(loggingContext, path), loggingContext => Logger.Log.EndSavingChangeTracker(loggingContext))) { Stopwatch sw = Stopwatch.StartNew(); FileEnvelope.WriteHeader(stream, atomicSaveToken); using (var writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false)) { if (IsDisabledOrNullTrackingSet) { writer.Write(true); } else { writer.Write(false); writer.Write(m_buildEngineFingerprint, (w, s) => w.Write(s)); m_changeTrackingSet.Save(writer); } } FileEnvelope.FixUpHeader(stream, atomicSaveToken); Logger.Log.SavingChangeTracker( pm.LoggingContext, path, atomicSaveToken.ToString(), m_changeTrackingSet == null ? "Null" : TrackingState.ToString(), m_changeTrackingSet == null ? 0 : m_changeTrackingSet.TrackedVolumes.Count(), sw.ElapsedMilliseconds); } }, ex => { throw new BuildXLException("Failed to save file change tracker", ex); }); } }
public override async Task <SandboxedProcessResult> GetResultAsync() { Contract.Requires(m_processExecutor != null); // (1) Wait for VmCommandProxy. await m_processExecutor.WaitForExitAsync(); await m_processExecutor.WaitForStdOutAndStdErrAsync(); // (2) Validate result of VmCommandProxy. if (m_processExecutor.TimedOut || m_processExecutor.Killed) { // If timed out/killed, then sandboxed process result may have not been deserialized yet. return(CreateResultForVmCommandProxyFailure()); } if (Process.ExitCode != 0) { return(CreateResultForVmCommandProxyFailure()); } if (!FileUtilities.FileExistsNoFollow(RunOutputPath)) { m_error.AppendLine($"Could not find VM output file '{RunOutputPath}"); return(CreateResultForVmCommandProxyFailure()); } try { // (3) Validate the result of sandboxed process executor run by VmCommandProxy. RunResult runVmResult = ExceptionUtilities.HandleRecoverableIOException( () => VmSerializer.DeserializeFromFile <RunResult>(RunOutputPath), e => m_error.AppendLine(e.Message)); if (runVmResult == null) { return(CreateResultForVmCommandProxyFailure()); } if (runVmResult.ProcessStateInfo.ExitCode != 0) { return(CreateResultForSandboxExecutorFailure(runVmResult)); } } catch (Exception e) { m_error.AppendLine(e.ToString()); return(CreateResultForVmCommandProxyFailure()); } return(DeserializeSandboxedProcessResultFromFile()); }
private Task <Possible <ContentHash, Failure> > TryStoreInternalAsync( ExpandedAbsolutePath path, FileRealizationMode fileRealizationModes, ContentHash?knownContentHash) { return(Task.Run <Possible <ContentHash, Failure> >( () => { lock (m_lock) { byte[] contentBytes = ExceptionUtilities.HandleRecoverableIOException( () => { return File.ReadAllBytes(path.ExpandedPath); }, ex => { throw new BuildXLException("Failed to store content (couldn't read new content from disk)", ex); }); ContentHash contentHash = ContentHashingUtilities.HashBytes(contentBytes); if (knownContentHash.HasValue && contentHash != knownContentHash.Value) { return new Failure <string>(I($"Stored content had an unexpected hash. (expected: {knownContentHash.Value}; actual: {contentHash})")); } CacheEntry entry; if (m_content.TryGetValue(contentHash, out entry)) { // We assume that stores of content already present somewhere still cause replication // to both the local and remote sites. See class remarks. entry.Sites |= CacheSites.LocalAndRemote; return contentHash; } else { try { if (m_pathRealizationModes != null) { m_pathRealizationModes[path.ExpandedPath] = fileRealizationModes; } // We assume that stored content is instantly and magically replicated to some remote place. // See class remarks. m_content[contentHash] = new CacheEntry(contentBytes, CacheSites.LocalAndRemote); return contentHash; } catch (BuildXLException ex) { return new RecoverableExceptionFailure(ex); } } } })); }
private bool CreateAndPreventDeletionHelper(string path) { Contract.Requires(!string.IsNullOrEmpty(path)); ExceptionUtilities.HandleRecoverableIOException( () => { Directory.CreateDirectory(path); }, ex => { throw new BuildXLException(I($"Failed to create the directory '{path}'"), ex); }); return(LockDirectory(path, false)); }
private static Possible <string> TryReadCacheConfigFile(string path) { try { return(ExceptionUtilities.HandleRecoverableIOException( () => File.ReadAllText(path), ex => { throw new BuildXLException("Unable to read cache configuration", ex); })); } catch (BuildXLException ex) { return(new RecoverableExceptionFailure(ex)); } }
private byte[] CalculateBytes(Stream content) { byte[] contentBytes = ExceptionUtilities.HandleRecoverableIOException( () => { MemoryStream memoryStream; Stream streamToRead; if (!content.CanSeek) { memoryStream = new MemoryStream(); streamToRead = memoryStream; } else { memoryStream = null; streamToRead = content; } using (memoryStream) { if (memoryStream != null) { content.CopyTo(memoryStream); memoryStream.Position = 0; } Contract.Assert(streamToRead.CanSeek); Contract.Assume(streamToRead.Length <= int.MaxValue); var length = (int)streamToRead.Length; var contentBytesLocal = new byte[length]; int read = 0; while (read < length) { int readThisIteration = streamToRead.Read(contentBytesLocal, read, length - read); if (readThisIteration == 0) { throw new BuildXLException("Unexpected end of stream"); } read += readThisIteration; } return(contentBytesLocal); } }, ex => throw new BuildXLException("Undable to calculate bytes from stream")); return(contentBytes); }
/// <inheritdoc /> public Task <Possible <ContentAvailabilityBatchResult, Failure> > TryLoadAvailableContentAsync(IReadOnlyList <ContentHash> hashes) { return(Task.Run <Possible <ContentAvailabilityBatchResult, Failure> >( () => { lock (m_lock) { bool allAvailable = true; var results = new ContentAvailabilityResult[hashes.Count]; for (int i = 0; i < hashes.Count; i++) { CacheEntry entry; bool available = m_content.TryGetValue(hashes[i], out entry) && entry.Sites != CacheSites.None; long bytesTransferredRemotely = 0; if (available) { if ((entry.Sites & CacheSites.Local) == 0) { ExceptionUtilities.HandleRecoverableIOException( () => { // Copy to local side. bytesTransferredRemotely = Download(hashes[i]); }, ex => throw new BuildXLException(I($"Failed to load content '{hashes[i]}' from remote site"), ex)); } entry.Sites |= CacheSites.Local; } results[i] = new ContentAvailabilityResult( hashes[i], isAvailable: available, bytesTransferred: bytesTransferredRemotely, sourceCache: nameof(MockArtifactContentCache)); allAvailable &= available; } return new ContentAvailabilityBatchResult( ReadOnlyArray <ContentAvailabilityResult> .FromWithoutCopy(results), allContentAvailable: allAvailable); } })); }
private void HandleRecoverableIOException(Action action) { try { ExceptionUtilities.HandleRecoverableIOException( action, ex => { throw new BuildXLException("Writing file failed", ex); }); } catch (BuildXLException ex) { m_exception = ex; ReleaseTextWriter(); ReleaseStringBuilder(); m_fileName = null; m_length = SandboxedProcessOutput.NoLength; IsFrozen = true; } }
private void ProcessQueueItem(Pip pip, MutablePipState pipState) { ExceptionUtilities.HandleRecoverableIOException( () => { var start = Stopwatch.GetTimestamp(); PageableStoreId value = m_store.Write(writer => ((PipWriter)writer).Write(pip)); #if DEBUG m_store.Read <Pip>(value, reader => ((PipReader)reader).ReadPip()); #endif pipState.StoreId = value; GC.KeepAlive(pip); // Pip must not get GCed until after StoreId has been set. var end = Stopwatch.GetTimestamp(); Interlocked.Add(ref m_writeTicks, end - start); Interlocked.Increment(ref m_writes); }, ex => ExceptionHandling.OnFatalException(ex)); }
private bool TryReadSandboxedProcessInfo(out SandboxedProcessInfo sandboxedProcessInfo) { sandboxedProcessInfo = null; sandboxedProcessInfo = ExceptionUtilities.HandleRecoverableIOException( () => { using (FileStream stream = File.OpenRead(Path.GetFullPath(m_configuration.SandboxedProcessInfoInputFile))) { // TODO: Custom DetoursEventListener? return(SandboxedProcessInfo.Deserialize(stream, m_loggingContext, detoursEventListener: null)); } }, ex => { m_logger.LogError(ex.ToString()); }); return(sandboxedProcessInfo != null); }
/// <inheritdoc /> public Task MoveFileAsync( string source, string destination, bool replaceExisting = false) { Contract.Requires(!string.IsNullOrEmpty(source)); Contract.Requires(!string.IsNullOrEmpty(destination)); return(Task.Run( () => ExceptionUtilities.HandleRecoverableIOException( () => { if (replaceExisting) { DeleteFile(destination); } File.Move(source, destination); }, ex => { throw new BuildXLException(I($"File move from '{source}' to '{destination}' failed"), ex); }))); }
private void EnsureRemote(ContentHash hash) { bool entryExists = m_content.TryGetValue(hash, out CacheEntry entry); Contract.Assert(entryExists); Contract.Assert(entry.Sites != CacheSites.None); GetPaths(hash, out string localPath, out string remotePath); if ((entry.Sites & CacheSites.Remote) == 0) { Contract.Requires((entry.Sites & CacheSites.Local) != 0); ExceptionUtilities.HandleRecoverableIOException( () => Upload(hash), ex => throw new BuildXLException(I($"Unable to ensure remote replication of content hash '{hash.ToString()}'"))); } else { Contract.Assert(File.Exists(remotePath)); } entry.Sites |= CacheSites.Remote; }
private static void SaveEntries(PathTable pathTable, AbsolutePath destination, List <Entry> entries) { var destinationPath = destination.ToString(pathTable); ExceptionUtilities.HandleRecoverableIOException( () => { // Build the report content. Materializes to the file on a per line basis since the whole content may be very big. using (var stream = new StreamWriter(destinationPath)) { stream.WriteLine(GetHeader()); foreach (var entry in entries) { stream.WriteLine(GetLine(entry)); } } }, ex => { throw new BuildXLException("Error while producing the report. Inner exception reason: " + ex.Message, ex); }); }
/// <summary> /// Reads the entire value; don't call this if the length is excessive, as it might OOM. /// </summary> /// <exception cref="BuildXLException">Thrown if a recoverable error occurs while opening the stream.</exception> public async Task <string> ReadValueAsync() { if (m_exception != null) { ExceptionDispatchInfo.Capture(m_exception).Throw(); } if (m_value != null) { return(m_value); } return(await ExceptionUtilities.HandleRecoverableIOException( async() => { using (TextReader reader = CreateFileReader()) { string value = await reader.ReadToEndAsync(); return value; } }, e => throw new BuildXLException("Failed to read a value from a stream", e))); }
/// <summary> /// Returns a new <see cref="FileStream" /> with the given creation mode, access level, and sharing. /// </summary> /// <remarks> /// This factory exists purely because FileStream cannot be constructed in async mode without also specifying a buffer /// size. /// We go ahead and wrap recoverable errors as BuildXLExceptions as well. /// /// Does not support paths longer than MAX_PATH /// </remarks> /// <exception cref="BuildXLException"> /// Thrown if a recoverable error occurs while opening the stream (see /// <see cref="ExceptionUtilities.HandleRecoverableIOException{T}" />) /// </exception> /// <param name="path">Path of file</param> /// <param name="fileMode">FileMode</param> /// <param name="fileAccess">FileAccess</param> /// <param name="fileShare">FileShare</param> /// <param name="options">FileOptions</param> /// <param name="force">Clears the readonly attribute if necessary</param> /// <param name="allowExcludeFileShareDelete">Indicates willful omission of FILE_SHARE_DELETE.</param> public static FileStream CreateFileStream( string path, FileMode fileMode, FileAccess fileAccess, FileShare fileShare, FileOptions options = FileOptions.None, bool force = false, bool allowExcludeFileShareDelete = false) { Contract.Requires(allowExcludeFileShareDelete || ((fileShare & FileShare.Delete) != 0)); Contract.EnsuresOnThrow <BuildXLException>(true); // The bufferSize of 4096 bytes is the default as used by the other FileStream constructors // http://index/mscorlib/system/io/filestream.cs.html return(ExceptionUtilities.HandleRecoverableIOException( () => { try { return new FileStream(path, fileMode, fileAccess, fileShare, bufferSize: DefaultBufferSize, options: options); } catch (UnauthorizedAccessException) { // This is a workaround to allow write access to a file that is marked as readonly. It is // exercised when hashing the output files of pips that create readonly files. The hashing currently // opens files as write if (force) { File.SetAttributes(path, File.GetAttributes(path) & ~FileAttributes.ReadOnly); return new FileStream(path, fileMode, fileAccess, fileShare, bufferSize: DefaultBufferSize, options: options); } throw; } }, ex => { throw new BuildXLException(string.Format(CultureInfo.InvariantCulture, "Failed to open path '{0}'", path), ex); })); }
/// <inheritdoc /> public FileStream CreateFileStream( string path, FileMode fileMode, FileAccess fileAccess, FileShare fileShare, FileOptions options, bool force) { // The bufferSize of 4096 bytes is the default as used by the other FileStream constructors // http://index/mscorlib/system/io/filestream.cs.html return(ExceptionUtilities.HandleRecoverableIOException( () => { try { return new FileStream(path, fileMode, fileAccess, fileShare, bufferSize: DefaultBufferSize, options: options); } catch (UnauthorizedAccessException) { // This is a workaround to allow write access to a file that is marked as readonly. It is // exercised when hashing the output files of pips that create readonly files. The hashing currently // opens files as write if (force) { File.SetAttributes(path, File.GetAttributes(path) & ~FileAttributes.ReadOnly); return new FileStream(path, fileMode, fileAccess, fileShare, bufferSize: DefaultBufferSize, options: options); } throw; } }, ex => { throw new BuildXLException(I($"Failed to open path '{path}' with mode='{fileMode}', access='{fileAccess}', share='{fileShare}'"), ex); })); }
private bool TryWriteSandboxedProcessResult(SandboxedProcessResult result) { Contract.Requires(result != null); bool success = false; ExceptionUtilities.HandleRecoverableIOException( () => { using (FileStream stream = File.OpenWrite(Path.GetFullPath(m_configuration.SandboxedProcessResultOutputFile))) { result.Serialize(stream); } success = true; }, ex => { m_logger.LogError(ex.ToString()); success = false; }); return(success); }
private void SaveInternal(string fileContentTablePath) { Contract.Requires(!string.IsNullOrWhiteSpace(fileContentTablePath)); Contract.EnsuresOnThrow <BuildXLException>(true); ExceptionUtilities.HandleRecoverableIOException( () => { int numEvicted = 0; Directory.CreateDirectory(Path.GetDirectoryName(fileContentTablePath)); // Note that we are using a non-async file stream here. That's because we're doing lots of tiny writes for simplicity, // but tiny writes on an async stream end up blocking anyway while adding silly overhead. using (FileStream stream = FileUtilities.CreateFileStream( fileContentTablePath, FileMode.Create, FileAccess.Write, FileShare.Delete, // Do not write the file with SequentialScan since it will be reread in the subsequent build FileOptions.None)) { // We don't have anything in particular to correlate this file to, // so we are simply creating a unique correlation id that is used as part // of the header consistency check. FileEnvelopeId correlationId = FileEnvelopeId.Create(); s_fileEnvelope.WriteHeader(stream, correlationId); using (var writer = new BuildXLWriter(debug: false, stream: stream, leaveOpen: true, logStats: false)) { long numberOfEntriesPosition = writer.BaseStream.Position; writer.Write(0U); uint entriesWritten = 0; var hashBuffer = new byte[ContentHashingUtilities.HashInfo.ByteLength]; foreach (var fileAndEntryPair in m_entries) { // Skip saving anything with a TTL of zero. These entries were loaded // with a TTL of one (immediately decremented) and were not used since load. // See class remarks. if (fileAndEntryPair.Value.TimeToLive == 0) { numEvicted++; continue; } // Key: Volume and File ID fileAndEntryPair.Key.Serialize(writer); // Entry: USN, hash, time to live. writer.Write(fileAndEntryPair.Value.Usn.Value); fileAndEntryPair.Value.Hash.SerializeHashBytes(hashBuffer, 0); writer.Write(hashBuffer); writer.Write(fileAndEntryPair.Value.Length); writer.Write(fileAndEntryPair.Value.TimeToLive); entriesWritten++; } var endPosition = writer.BaseStream.Position; writer.BaseStream.Position = numberOfEntriesPosition; writer.Write(entriesWritten); writer.BaseStream.Position = endPosition; } s_fileEnvelope.FixUpHeader(stream, correlationId); } Counters.AddToCounter(FileContentTableCounters.NumEvicted, numEvicted); return(Unit.Void); }, ex => { throw new BuildXLException("Failure writing file content table", ex); }); }
/// <summary> /// Attempts to create auto-complete items from the "tagged template" statements for file, path, directory and relative path (f``, d``, p``) /// </summary> internal static IEnumerable <CompletionItem> TryCreateFileCompletionItemsForTaggedTemplatedExpression(CompletionState completionState, INode completionNode) { var templateExpression = completionNode.Cast <ITaggedTemplateExpression>(); // GetTemplateText does a Cast on the expression and then access the "text" property // instead of just returning a null or empty string. // So, before we can use GetTemplateText, we must check to see if it is actually of // the correct type. // We cannot use the syntax kind, as ILiteralExpression can cover many syntax kinds - // Strings, numbers, etc. var literalExprssion = templateExpression?.TemplateExpression?.As <ILiteralExpression>(); var existingFileExpressionText = literalExprssion?.GetTemplateText(); if (!string.IsNullOrEmpty(existingFileExpressionText)) { if (!RelativePath.TryCreate(completionState.PathTable.StringTable, existingFileExpressionText, out var normalizedPath)) { return(null); } existingFileExpressionText = normalizedPath.ToString(completionState.PathTable.StringTable); } else { existingFileExpressionText = string.Empty; } var sourceFileRootPath = Path.GetDirectoryName(completionState.StartingNode.SourceFile.ToUri().LocalPath); // Now, for this bit of fun. If the user has started typing a path already, then we still // want to complete what they have started typing. // If what they have typed does not have a directory separator in it, then we will use // that as the search pattern below. If it does however, then we will combine that // with the spec directory and then attempt to get the remainder to use as the search path. // So... if the user has typed `foo` we will use the spec path as the root search directory // and `foo*` as the search pattern. // If the user has typed `foo\bar` then we will use `<spec path>\foo` as the search path and // "bar" as the search pattern. var originalSourceFileRootPath = sourceFileRootPath; if (existingFileExpressionText.Contains(Path.DirectorySeparatorChar)) { sourceFileRootPath = Path.Combine(sourceFileRootPath, existingFileExpressionText); existingFileExpressionText = Path.GetFileName(sourceFileRootPath); sourceFileRootPath = Path.GetDirectoryName(sourceFileRootPath); } // If the user types just a "\" character, then the source file root path can be null if (string.IsNullOrEmpty(sourceFileRootPath)) { return(null); } // If we managed to get a path that is outside of our spec path, then bail. Do not allow auto completion if (!sourceFileRootPath.StartsWith(originalSourceFileRootPath, System.StringComparison.OrdinalIgnoreCase) || !Directory.Exists(sourceFileRootPath)) { return(null); } bool isDirectoryTag = templateExpression.GetInterpolationKind() == InterpolationKind.DirectoryInterpolation; // Leverage BuildXL's recoverable IO exception extension to ensure that if we hit // cases like "access denied" that it does not take down the plugin. IEnumerable <string> fileSystemEntries = null; try { fileSystemEntries = ExceptionUtilities.HandleRecoverableIOException( () => { return(isDirectoryTag ? Directory.EnumerateDirectories(sourceFileRootPath, existingFileExpressionText + "*", SearchOption.TopDirectoryOnly) : Directory.EnumerateFileSystemEntries(sourceFileRootPath, existingFileExpressionText + "*", SearchOption.TopDirectoryOnly)); }, ex => { throw new BuildXLException(ex.Message); }); } catch (BuildXLException) { } if (fileSystemEntries.IsNullOrEmpty()) { return(null); } // We aren't done yet :) // Let's see if we can filter further if we are in an array literal experssion.. // So, if the user has something like this // sources: [ // f`fileA.cpp`, // f`fileB.cpp`, // // And they have files fileC-FileZ.cpp on disk, we filter out fileA.cpp and fileB.cpp. // This is very similar to filtering out properties that are already set on an object // literal. if (templateExpression.Parent?.Kind == SyntaxKind.ArrayLiteralExpression) { var existingFiles = new List <string>(); var existingArrayLiteralExpression = templateExpression.Parent.Cast <IArrayLiteralExpression>(); foreach (var existingFileItem in existingArrayLiteralExpression.Elements) { if (existingFileItem.Kind == SyntaxKind.TaggedTemplateExpression) { var existingFileTemplateExpression = existingFileItem.Cast <ITaggedTemplateExpression>(); // GetTemplateText does a Cast on the expression and then access the "text" property // instead of just returning a null or empty string. // So, before we can use GetTemplateText, we must check to see if it is actually of // the correct type. // We cannot use the syntax kind, as ILiteralExpression can cover many syntax kinds - // Strings, numbers, etc. var existingFileLiteralExprssion = existingFileTemplateExpression?.TemplateExpression?.As <ILiteralExpression>(); var existingFilePathText = existingFileLiteralExprssion?.GetTemplateText(); if (!string.IsNullOrEmpty(existingFilePathText)) { existingFiles.Add(Path.Combine(originalSourceFileRootPath, existingFilePathText)); } } } fileSystemEntries = fileSystemEntries.Where(possibleEntry => !existingFiles.Contains(possibleEntry, StringComparer.InvariantCultureIgnoreCase)); } return(fileSystemEntries.Select(name => { var itemLabel = name.Substring(sourceFileRootPath.Length + 1); var item = new CompletionItem() { Kind = CompletionItemKind.File, Label = itemLabel, }; return item; })); }
private Task <Possible <ContentHash, Failure> > TryStoreInternalAsync( ExpandedAbsolutePath path, FileRealizationMode fileRealizationModes, ContentHash?knownContentHash) { return(Task.Run <Possible <ContentHash, Failure> >( () => { lock (m_lock) { byte[] contentBytes = ExceptionUtilities.HandleRecoverableIOException( () => { var expandedPath = path.ExpandedPath; using (FileStream fileStream = new FileStream(expandedPath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (BinaryReader binaryReader = new BinaryReader(fileStream)) { // This will work with files up to 2GB in length, due to the 'int' API signature return binaryReader.ReadBytes((int)new FileInfo(expandedPath).Length); } }, ex => { throw new BuildXLException("Failed to store content (couldn't read new content from disk)", ex); }); ContentHash contentHash = ContentHashingUtilities.HashBytes(contentBytes); if (knownContentHash.HasValue && contentHash != knownContentHash.Value) { return new Failure <string>(I($"Stored content had an unexpected hash. (expected: {knownContentHash.Value}; actual: {contentHash})")); } CacheEntry entry; if (m_content.TryGetValue(contentHash, out entry)) { // We assume that stores of content already present somewhere still cause replication // to both the local and remote sites. See class remarks. entry.Sites |= CacheSites.LocalAndRemote; return contentHash; } else { try { if (m_pathRealizationModes != null) { m_pathRealizationModes[path.ExpandedPath] = fileRealizationModes; } // We assume that stored content is instantly and magically replicated to some remote place. // See class remarks. m_content[contentHash] = new CacheEntry(contentBytes, CacheSites.LocalAndRemote); return contentHash; } catch (BuildXLException ex) { return new RecoverableExceptionFailure(ex); } } } })); }
public static LoadingTrackerResult TryLoad( LoggingContext loggingContext, string path, VolumeMap volumeMap, IChangeJournalAccessor journal, string buildEngineFingerprint, bool loadForAllCapableVolumes = true) { Contract.Requires(loggingContext != null); Contract.Requires(!loadForAllCapableVolumes || volumeMap != null); Contract.Requires(!loadForAllCapableVolumes || journal != null); Stopwatch stopwatch = Stopwatch.StartNew(); SafeFileHandle handle; OpenFileResult result = FileUtilities.TryCreateOrOpenFile( path, FileDesiredAccess.GenericRead, FileShare.Read | FileShare.Delete, FileMode.Open, // Ok to evict the file from standby since the file will be overwritten and never reread from disk after this point. FileFlagsAndAttributes.FileFlagSequentialScan, out handle); if (result.Succeeded) { Contract.Assume(handle != null); Contract.Assume(!handle.IsInvalid); using (handle) { using (var stream = new FileStream(handle, FileAccess.Read)) { FileEnvelopeId fileEnvelopeId; try { fileEnvelopeId = FileEnvelope.ReadHeader(stream); } catch (BuildXLException e) { return(LoadingTrackerResult.FailBadFormatMarker(e.Message)); } try { return(ExceptionUtilities.HandleRecoverableIOException( () => { using (var reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: true)) { bool wasTrackerDisabled = reader.ReadBoolean(); if (wasTrackerDisabled) { return LoadingTrackerResult.FailPriorTrackerDisabled(); } var previousFingerprint = reader.ReadNullable(r => r.ReadString()); // only check for fingerprints match if the supplied fingerprint is valid // this is to support special cases where we might want to load ChangeTracker // regardless of the previously stored fingerprint value if (buildEngineFingerprint != null && !string.Equals(previousFingerprint, buildEngineFingerprint, StringComparison.Ordinal)) { return LoadingTrackerResult.FailBuildEngineFingerprintMismatch(); } return FileChangeTrackingSet.TryLoad( loggingContext, fileEnvelopeId, reader, volumeMap, journal, stopwatch, loadForAllCapableVolumes); } }, ex => { throw new BuildXLException(ex.Message); })); } catch (Exception e) { // Catch any exception. Failure in loading FileChangeTracker should not // cause build break, or worse, make people stuck on erroneous state. // In such a case, BuildXL simply has to start tracking from scratch. return(LoadingTrackerResult.FailLoadException(e.GetLogEventMessage())); } } } } Contract.Assume(handle == null); return(LoadingTrackerResult.FailTrackingSetCannotBeOpened(result.CreateFailureForError().DescribeIncludingInnerFailures())); }
private Task <Possible <ContentHash, Failure> > TryStoreInternalAsync(Stream content, ContentHash?knownContentHash) { return(Task.Run <Possible <ContentHash, Failure> >( () => { lock (m_lock) { CacheEntry entry; if (knownContentHash.HasValue && m_content.TryGetValue(knownContentHash.Value, out entry)) { // We assume that stores of content already present somewhere still cause replication // to both the local and remote sites. See class remarks. entry.Sites |= CacheSites.LocalAndRemote; return knownContentHash.Value; } else { try { byte[] contentBytes = ExceptionUtilities.HandleRecoverableIOException( () => { MemoryStream memoryStream; Stream streamToRead; if (!content.CanSeek) { memoryStream = new MemoryStream(); streamToRead = memoryStream; } else { memoryStream = null; streamToRead = content; } using (memoryStream) { if (memoryStream != null) { content.CopyTo(memoryStream); memoryStream.Position = 0; } Contract.Assert(streamToRead.CanSeek); Contract.Assume(streamToRead.Length <= int.MaxValue); var length = (int)streamToRead.Length; var contentBytesLocal = new byte[length]; int read = 0; while (read < length) { int readThisIteration = streamToRead.Read(contentBytesLocal, read, length - read); if (readThisIteration == 0) { throw new BuildXLException("Unexpected end of stream"); } read += readThisIteration; } return contentBytesLocal; } }, ex => { throw new BuildXLException("Failed to read content from the provided stream in order to store it", ex); }); ContentHash contentHash = knownContentHash ?? ContentHashingUtilities.HashBytes(contentBytes); // We assume that stored content is instantly and magically replicated to some remote place. // See class remarks. m_content[contentHash] = new CacheEntry(contentBytes, CacheSites.LocalAndRemote); return contentHash; } catch (BuildXLException ex) { return new RecoverableExceptionFailure(ex); } } } })); }
private Task <Possible <ContentHash, Failure> > TryStoreInternalAsync( ExpandedAbsolutePath path, FileRealizationMode fileRealizationMode, ContentHash?knownContentHash) { return(Task.Run <Possible <ContentHash, Failure> >( () => { lock (m_lock) { string expandedPath = path.ExpandedPath; ContentHash contentHash; if (knownContentHash.HasValue) { contentHash = knownContentHash.Value; var mayBeHash = TryHashFile(expandedPath); if (!mayBeHash.Succeeded) { return mayBeHash.Failure; } if (contentHash != mayBeHash.Result) { return new Failure <string>(I($"Stored content had an unexpected hash. (expected: {contentHash}; actual: {mayBeHash.Result})")); } } else { var maybeHash = TryHashFile(expandedPath); if (!maybeHash.Succeeded) { return maybeHash.Failure; } contentHash = maybeHash.Result; } string localPath = GetLocalPath(contentHash); CacheEntry entry; if (m_content.TryGetValue(contentHash, out entry)) { EnsureLocal(contentHash); EnsureRemote(contentHash); return ExceptionUtilities.HandleRecoverableIOException <Possible <ContentHash, Failure> >( () => { var putFile = PutFileInternal(expandedPath, contentHash, fileRealizationMode); if (!putFile.Succeeded) { return putFile.Failure; } return contentHash; }, ex => throw new BuildXLException(I($"Failed to store '{expandedPath}'"), ex)); } else { var result = ExceptionUtilities.HandleRecoverableIOException <Possible <ContentHash, Failure> >( () => { var putFile = PutFileInternal(expandedPath, contentHash, fileRealizationMode); if (!putFile.Succeeded) { return putFile.Failure; } return contentHash; }, ex => throw new BuildXLException(I($"Failed to store '{expandedPath}'"), ex)); m_content.Add(contentHash, new CacheEntry(CacheSites.Local)); EnsureRemote(contentHash); m_pathRealizationModes[expandedPath] = fileRealizationMode; return result; } } })); }