/// <summary> /// Serialize hibernated session information to the standard filename in the given directory. /// </summary> public static async Task WriteProtectedAsync <TInfo>(this HibernatedSessions <TInfo> sessions, IAbsFileSystem fileSystem, AbsolutePath rootPath, string fileName) { Contract.Requires(rootPath != null); // Due to abnormal process termination, the file that we'll be writing can be corrupted. // To prevent this issue we first write the file into a temporary location and then we "move it" into a final location. using (var tempFolder = new DisposableDirectory(fileSystem, rootPath / "Temp")) { var jsonTempPath = tempFolder.CreateRandomFileName(); var jsonPath = rootPath / fileName; using (var memoryStream = new MemoryStream()) { sessions.SerializeToJSON(memoryStream); var bytes = memoryStream.ToArray(); var protectedBytes = ProtectedData.Protect(bytes, optionalEntropy: null, DataProtectionScope.CurrentUser); using var fileStream = fileSystem.Open(jsonTempPath, FileAccess.Write, FileMode.Create, FileShare.None); await fileStream.Stream.WriteAsync(protectedBytes, 0, protectedBytes.Length); } fileSystem.MoveFile(jsonTempPath, jsonPath, replaceExisting: true); } }
private static BoolResult SaveQuota(IAbsFileSystem fileSystem, AbsolutePath rootPath, MaxSizeQuota quota, long historyTimestampInTick) { var filePath = rootPath / BinaryFileName; try { fileSystem.DeleteFile(filePath); using (var stream = fileSystem.Open(filePath, FileAccess.Write, FileMode.CreateNew, FileShare.Delete)) { using (var writer = new BinaryWriter(stream)) { writer.Write(quota.Hard); writer.Write(quota.Soft); writer.Write(historyTimestampInTick); } } return(BoolResult.Success); } catch (Exception e) { // When failed, clean up so that it is not used in the next load. fileSystem.DeleteFile(filePath); return(new BoolResult(e)); } }
/// <summary> /// Saves this instance to disk. /// </summary> public Task SaveAsync(IAbsFileSystem fileSystem) { Contract.Requires(fileSystem != null); var filePath = _directoryPath / BinaryFileName; try { fileSystem.DeleteFile(filePath); using (var stream = fileSystem.Open(filePath, FileAccess.Write, FileMode.CreateNew, FileShare.Delete)) { using (var writer = new BinaryWriter(stream)) { lock (_pinHistoryBuffer) { _pinHistoryBuffer.Serialize(writer); writer.Write(_timestampInTick); } } } } catch (IOException) { // When failed, clean up so that it is not used in the next load. fileSystem.DeleteFile(filePath); } return(Task.CompletedTask); }
/// <summary> /// Calculate content hash of content in a file. /// </summary> /// <exception cref="FileNotFoundException">Throws if the file <paramref name="path"/> is not on disk.</exception> public static async Task <ContentHash> CalculateHashAsync(this IAbsFileSystem fileSystem, AbsolutePath path, HashType hashType) { using (var stream = fileSystem.Open( path, FileAccess.Read, FileMode.Open, FileShare.Read | FileShare.Delete, FileOptions.SequentialScan, HashStreamBufferSize)) { return(await stream.CalculateHashAsync(hashType)); } }
private Task <Result <LogFile> > WriteLogsToFileAsync(OperationContext context, AbsolutePath logFilePath, string[] logs) { return(context.PerformOperationAsync(Tracer, async() => { long compressedSizeBytes = 0; long uncompressedSizeBytes = 0; using (Stream fileStream = _fileSystem.Open( logFilePath, FileAccess.Write, FileMode.CreateNew, FileShare.None, FileOptions.SequentialScan | FileOptions.Asynchronous)) { // We need to make sure we close the compression stream before we take the fileStream's // position, because the compression stream won't write everything until it's been closed, // which leads to bad recorded values in compressedSizeBytes. using (var gzipStream = new GZipStream(fileStream, CompressionLevel.Fastest, leaveOpen: true)) { using var recordingStream = new CountingStream(gzipStream); using var streamWriter = new StreamWriter(recordingStream, Encoding.UTF8, bufferSize: 32 * 1024, leaveOpen: true); if (OnFileOpen != null) { await OnFileOpen(streamWriter); } foreach (var log in logs) { await streamWriter.WriteLineAsync(log); } if (OnFileClose != null) { await OnFileClose(streamWriter); } // Needed to ensure the recording stream receives everything it needs to receive await streamWriter.FlushAsync(); uncompressedSizeBytes = recordingStream.BytesWritten; } compressedSizeBytes = fileStream.Position; } Tracer.TrackMetric(context, $"LogLinesWritten", logs.Length); Tracer.TrackMetric(context, $"CompressedBytesWritten", compressedSizeBytes); Tracer.TrackMetric(context, $"UncompressedBytesWritten", uncompressedSizeBytes); return new Result <LogFile>(new LogFile() { Path = logFilePath, UncompressedSizeBytes = uncompressedSizeBytes, CompressedSizeBytes = compressedSizeBytes, }); },
private async Task <bool> ValidateNameHashesMatchContentHashesAsync(Context context) { int mismatchedParentDirectoryCount = 0; int mismatchedContentHashCount = 0; _tracer.Always(context, "Validating local CAS content hashes..."); await TaskUtilities.SafeWhenAll(_enumerateBlobPathsFromDisk().Select( async blobPath => { var contentFile = blobPath.FullPath; if (!contentFile.FileName.StartsWith(contentFile.GetParent().FileName, StringComparison.OrdinalIgnoreCase)) { mismatchedParentDirectoryCount++; _tracer.Debug( context, $"The first {FileSystemContentStoreInternal.HashDirectoryNameLength} characters of the name of content file at {contentFile}" + $" do not match the name of its parent directory {contentFile.GetParent().FileName}."); } if (!FileSystemContentStoreInternal.TryGetHashFromPath(context, _tracer, contentFile, out var hashFromPath)) { _tracer.Debug( context, $"The path '{contentFile}' does not contain a well-known hash name."); return; } var hasher = HashInfoLookup.GetContentHasher(hashFromPath.HashType); ContentHash hashFromContents; using (var contentStream = _fileSystem.Open( contentFile, FileAccess.Read, FileMode.Open, FileShare.Read | FileShare.Delete, FileOptions.SequentialScan, HashingExtensions.HashStreamBufferSize)) { hashFromContents = await hasher.GetContentHashAsync(contentStream); } if (hashFromContents != hashFromPath) { mismatchedContentHashCount++; _tracer.Debug( context, $"Content at {contentFile} content hash {hashFromContents.ToShortString()} did not match expected value of {hashFromPath.ToShortString()}."); } })); _tracer.Always(context, $"{mismatchedParentDirectoryCount} mismatches between content file name and parent directory."); _tracer.Always(context, $"{mismatchedContentHashCount} mismatches between content file name and file contents."); return(mismatchedContentHashCount == 0 && mismatchedParentDirectoryCount == 0); }
/// <inheritdoc /> public async Task <Possible <string, Failure> > ShutdownAsync() { Contract.Requires(!IsShutdown); m_isShutdown = true; try { try { GetStatsResult stats = await m_cache.GetStatsAsync(new Context(m_logger)); if (stats.Succeeded) { using (Stream fileStream = m_fileSystem.Open(m_statsFile, FileAccess.ReadWrite, FileMode.CreateNew, FileShare.None)) { using (StreamWriter sw = new StreamWriter(fileStream)) { foreach (KeyValuePair <string, long> stat in stats.CounterSet.ToDictionaryIntegral()) { await sw.WriteLineAsync($"{stat.Key}={stat.Value}"); } } } } else { m_logger.Debug($"Stats call failed {stats.ErrorMessage}"); } } #pragma warning disable ERP022 // Unobserved exception in generic exception handler catch { } #pragma warning restore ERP022 // Unobserved exception in generic exception handler BoolResult shutdownResult = await m_cache.ShutdownAsync(new Context(m_logger)); if (shutdownResult.Succeeded) { return(CacheId.ToString()); } return(new CacheFailure(shutdownResult.ErrorMessage)); } finally { Dispose(); } }
/// <summary> /// Opens a file for hashing purposes. /// </summary> public static StreamWithLength OpenForHashing(this IAbsFileSystem fileSystem, AbsolutePath path) { // Using a helper from the hashing layer that will pass the right options required for file hashing. return(ContentHashingHelper.OpenForHashing(path.Path, tuple => fileSystem.Open( path, tuple.fileAccess, tuple.mode, tuple.fileShare, tuple.options, #if NET_COREAPP tuple.bufferSize).ToFileStream())); #else FileSystemDefaults.DefaultFileStreamBufferSize).ToFileStream()); #endif }
/// <summary> /// Serialize a ContentStoreConfiguration to JSON in the standard filename in a CAS root directory. /// </summary> public static void Write(this ContentStoreConfiguration configuration, IAbsFileSystem fileSystem, AbsolutePath rootPath) { Contract.Requires(rootPath != null); Contract.Requires(configuration != null); Contract.Requires(configuration.IsValid); AbsolutePath jsonPath = rootPath / FileName; if (!fileSystem.DirectoryExists(rootPath)) { throw new CacheException($"Directory path=[{rootPath}] does not exist"); } using (var stream = fileSystem.Open(jsonPath, FileAccess.Write, FileMode.Create, FileShare.None)) { configuration.SerializeToJSON(stream); } }
/// <summary> /// Serialize hibernated session information to the standard filename in the given directory. /// </summary> public static void Write <TInfo>(this HibernatedSessions <TInfo> sessions, IAbsFileSystem fileSystem, AbsolutePath rootPath, string fileName) { Contract.Requires(rootPath != null); // Due to abnormal process termination, the file that we'll be writing can be corrupted. // To prevent this issue we first write the file into a temporary location and then we "move it" into a final location. using (var tempFolder = new DisposableDirectory(fileSystem, rootPath / "Temp")) { var jsonTempPath = tempFolder.CreateRandomFileName(); var jsonPath = rootPath / fileName; using (var stream = fileSystem.Open(jsonTempPath, FileAccess.Write, FileMode.Create, FileShare.None)) { sessions.SerializeToJSON(stream); } fileSystem.MoveFile(jsonTempPath, jsonPath, replaceExisting: true); } }
/// <summary> /// Writes the content to a file <paramref name="absolutePath"/>. /// </summary> /// <exception cref="Exception">Throws if the IO operation fails.</exception> public static void WriteAllText(this IAbsFileSystem fileSystem, AbsolutePath absolutePath, string contents, FileShare fileShare = FileShare.ReadWrite) { using Stream file = fileSystem.Open(absolutePath, FileAccess.Write, FileMode.Create, fileShare); using var writer = new StreamWriter(file); writer.Write(contents); }
/// <summary> /// Reads the content from a file <paramref name="absolutePath"/>. /// </summary> /// <exception cref="Exception">Throws if the IO operation fails.</exception> public static string ReadAllText(this IAbsFileSystem fileSystem, AbsolutePath absolutePath, FileShare fileShare = FileShare.ReadWrite) { using Stream readLockFile = fileSystem.Open(absolutePath, FileAccess.Read, FileMode.Open, fileShare); using var reader = new StreamReader(readLockFile); return(reader.ReadToEnd()); }
/// <summary> /// AcquireAsync the lock, waiting as long as it takes or until the configured timeout. /// </summary> public async Task <LockAcquisitionResult> AcquireAsync(Context context, TimeSpan waitTimeout) { _tracer.Info(context, $"Acquiring lock file=[{_lockFilePath}]"); _fileSystem.CreateDirectory(_lockFilePath.GetParent()); DateTime timeOutTime = DateTime.UtcNow + waitTimeout; Exception?lastException = null; int? lastCompetingProcessId = null; while (DateTime.UtcNow < timeOutTime) { try { // Anything other than FileShare.None is effectively ignored in Unix FileShare fileShare = BuildXL.Utilities.OperatingSystemHelper.IsUnixOS ? FileShare.None : FileShare.Read; _lockFile = _fileSystem.Open(_lockFilePath, FileAccess.Write, FileMode.OpenOrCreate, fileShare); using (var writer = new StreamWriter(_lockFile, UTF8WithoutBom, bufferSize: 4096, leaveOpen: true)) { await writer.WriteLineAsync( $"Lock acquired at {DateTime.UtcNow:O} by computer [{Environment.MachineName}] running command line [{Environment.CommandLine}] with process id [{Process.GetCurrentProcess().Id}]" ); } _tracer.Info(context, $"Acquired lock file=[{_lockFilePath}]"); await _lockFile.FlushAsync(); return(LockAcquisitionResult.Acquired()); } catch (IOException ioException) { lastException = ioException; } catch (UnauthorizedAccessException accessException) { lastException = accessException; } try { string?contents = await _fileSystem.TryReadFileAsync(_lockFilePath); if (contents != null) { _tracer.Diagnostic(context, $"Lock file=[{_lockFilePath}] contains [{contents}]"); lastCompetingProcessId = TryExtractProcessIdFromLockFilesContent(contents); } } catch (Exception readLockFileException) { string message = readLockFileException is UnauthorizedAccessException ae ? ae.Message : readLockFileException.ToString(); // This is just extra cautious. We shouldn't fail hard being unable to get this diagnostic information. _tracer.Info( context, $"Unable to read contents of lock file=[{_lockFilePath}] because [{message}]"); } await Task.Delay(_pollingInterval); } string lastProcessIdText = lastCompetingProcessId == null ? string.Empty : " Competing process Id: " + lastCompetingProcessId; _tracer.Info( context, $"Timed out trying to acquire lock file=[{_lockFilePath}].{lastProcessIdText} Last exception was=[{lastException}]"); return(LockAcquisitionResult.Failed(waitTimeout, lastCompetingProcessId, TryGetProcessName(lastCompetingProcessId), lastException)); }