private static async Task <BoolResult> SaveQuotaAsync(IAbsFileSystem fileSystem, AbsolutePath rootPath, MaxSizeQuota quota, long historyTimestampInTick)
{
var filePath = rootPath / BinaryFileName;
try
{
fileSystem.DeleteFile(filePath);
using (
var stream =
await fileSystem.OpenSafeAsync(filePath, FileAccess.Write, FileMode.CreateNew, FileShare.Delete))
{
using (var writer = new BinaryWriter(stream))
{
writer.Write(quota.Hard);
writer.Write(quota.Soft);
writer.Write(historyTimestampInTick);
}
}
return(BoolResult.Success);
}
catch (Exception e)
{
// When failed, clean up so that it is not used in the next load.
fileSystem.DeleteFile(filePath);
return(new BoolResult(e));
}
}
/// <summary>
/// Serialize hibernated session information to the standard filename in the given directory.
/// </summary>
public static async Task WriteProtectedAsync <TInfo>(this HibernatedSessions <TInfo> sessions, IAbsFileSystem fileSystem, AbsolutePath rootPath, string fileName)
{
Contract.Requires(fileSystem != null);
Contract.Requires(rootPath != null);
// Due to abnormal process termination, the file that we'll be writing can be corrupted.
// To prevent this issue we first write the file into a temporary location and then we "move it" into a final location.
using (var tempFolder = new DisposableDirectory(fileSystem, rootPath / "Temp"))
{
var jsonTempPath = tempFolder.CreateRandomFileName();
var jsonPath = rootPath / fileName;
using (var memoryStream = new MemoryStream())
{
sessions.SerializeToJSON(memoryStream);
var bytes = memoryStream.ToArray();
var protectedBytes = ProtectedData.Protect(bytes, optionalEntropy: null, DataProtectionScope.CurrentUser);
using var fileStream = await fileSystem.OpenSafeAsync(jsonTempPath, FileAccess.Write, FileMode.Create, FileShare.None);
await fileStream.Stream.WriteAsync(protectedBytes, 0, protectedBytes.Length);
}
fileSystem.MoveFile(jsonTempPath, jsonPath, replaceExisting: true);
}
}
/// <summary>
/// Saves this instance to disk.
/// </summary>
public async Task SaveAsync(IAbsFileSystem fileSystem)
{
Contract.Requires(fileSystem != null);
var filePath = _directoryPath / BinaryFileName;
try
{
fileSystem.DeleteFile(filePath);
using (
var stream =
await fileSystem.OpenSafeAsync(filePath, FileAccess.Write, FileMode.CreateNew, FileShare.Delete))
{
using (var writer = new BinaryWriter(stream))
{
lock (_pinHistoryBuffer)
{
_pinHistoryBuffer.Serialize(writer);
writer.Write(_timestampInTick);
}
}
}
}
catch (IOException)
{
// When failed, clean up so that it is not used in the next load.
fileSystem.DeleteFile(filePath);
}
}
/// <summary>
/// Calculate content hash of content in a file.
/// </summary>
/// <exception cref="FileNotFoundException">Throws if the file <paramref name="path"/> is not on disk.</exception>
public static async Task <ContentHash> CalculateHashAsync(this IAbsFileSystem fileSystem, AbsolutePath path, HashType hashType)
{
using (var stream = await fileSystem.OpenSafeAsync(
path, FileAccess.Read, FileMode.Open, FileShare.Read | FileShare.Delete, FileOptions.SequentialScan, HashStreamBufferSize))
{
return(await stream.CalculateHashAsync(hashType));
}
}
private Task <Result <LogFile> > WriteLogsToFileAsync(OperationContext context, AbsolutePath logFilePath, string[] logs)
{
return(context.PerformOperationAsync(Tracer, async() =>
{
long compressedSizeBytes = 0;
long uncompressedSizeBytes = 0;
using (Stream fileStream = await _fileSystem.OpenSafeAsync(
logFilePath,
FileAccess.Write,
FileMode.CreateNew,
FileShare.None,
FileOptions.SequentialScan | FileOptions.Asynchronous))
{
// We need to make sure we close the compression stream before we take the fileStream's
// position, because the compression stream won't write everything until it's been closed,
// which leads to bad recorded values in compressedSizeBytes.
using (var gzipStream = new GZipStream(fileStream, CompressionLevel.Fastest, leaveOpen: true))
{
using var recordingStream = new CountingStream(gzipStream);
using var streamWriter = new StreamWriter(recordingStream, Encoding.UTF8, bufferSize: 32 * 1024, leaveOpen: true);
if (OnFileOpen != null)
{
await OnFileOpen(streamWriter);
}
foreach (var log in logs)
{
await streamWriter.WriteLineAsync(log);
}
if (OnFileClose != null)
{
await OnFileClose(streamWriter);
}
// Needed to ensure the recording stream receives everything it needs to receive
await streamWriter.FlushAsync();
uncompressedSizeBytes = recordingStream.BytesWritten;
}
compressedSizeBytes = fileStream.Position;
}
Tracer.TrackMetric(context, $"LogLinesWritten", logs.Length);
Tracer.TrackMetric(context, $"CompressedBytesWritten", compressedSizeBytes);
Tracer.TrackMetric(context, $"UncompressedBytesWritten", uncompressedSizeBytes);
return new Result <LogFile>(new LogFile()
{
Path = logFilePath,
UncompressedSizeBytes = uncompressedSizeBytes,
CompressedSizeBytes = compressedSizeBytes,
});
},
/// <summary>
/// Writes the content to a file <paramref name="absolutePath"/>.
/// </summary>
/// <exception cref="Exception">Throws if the IO operation fails.</exception>
public static void WriteAllText(this IAbsFileSystem fileSystem, AbsolutePath absolutePath, string contents, FileShare fileShare = FileShare.ReadWrite)
{
using (Stream file = fileSystem.OpenSafeAsync(absolutePath, FileAccess.Write, FileMode.Create, fileShare).GetAwaiter().GetResult())
{
using (var writer = new StreamWriter(file))
{
writer.WriteLine(contents);
}
}
}
/// <summary>
/// Reads the content from a file <paramref name="absolutePath"/>.
/// </summary>
/// <exception cref="Exception">Throws if the IO operation fails.</exception>
public static string ReadAllText(this IAbsFileSystem fileSystem, AbsolutePath absolutePath, FileShare fileShare = FileShare.ReadWrite)
{
using (Stream readLockFile = fileSystem.OpenSafeAsync(absolutePath, FileAccess.Read, FileMode.Open, fileShare).GetAwaiter().GetResult())
{
using (var reader = new StreamReader(readLockFile))
{
return(reader.ReadToEnd());
}
}
}
private async Task <bool> ValidateNameHashesMatchContentHashesAsync(Context context)
{
int mismatchedParentDirectoryCount = 0;
int mismatchedContentHashCount = 0;
_tracer.Always(context, "Validating local CAS content hashes...");
await TaskSafetyHelpers.WhenAll(_enumerateBlobPathsFromDisk().Select(
async blobPath =>
{
var contentFile = blobPath.FullPath;
if (!contentFile.FileName.StartsWith(contentFile.GetParent().FileName, StringComparison.OrdinalIgnoreCase))
{
mismatchedParentDirectoryCount++;
_tracer.Debug(
context,
$"The first {FileSystemContentStoreInternal.HashDirectoryNameLength} characters of the name of content file at {contentFile}" +
$" do not match the name of its parent directory {contentFile.GetParent().FileName}.");
}
if (!FileSystemContentStoreInternal.TryGetHashFromPath(contentFile, out var hashFromPath))
{
_tracer.Debug(
context,
$"The path '{contentFile}' does not contain a well-known hash name.");
return;
}
var hasher = ContentHashers.Get(hashFromPath.HashType);
ContentHash hashFromContents;
using (var contentStream = await _fileSystem.OpenSafeAsync(
contentFile, FileAccess.Read, FileMode.Open, FileShare.Read | FileShare.Delete, FileOptions.SequentialScan, HashingExtensions.HashStreamBufferSize))
{
hashFromContents = await hasher.GetContentHashAsync(contentStream);
}
if (hashFromContents != hashFromPath)
{
mismatchedContentHashCount++;
_tracer.Debug(
context,
$"Content at {contentFile} content hash {hashFromContents.ToShortString()} did not match expected value of {hashFromPath.ToShortString()}.");
}
}));
_tracer.Always(context, $"{mismatchedParentDirectoryCount} mismatches between content file name and parent directory.");
_tracer.Always(context, $"{mismatchedContentHashCount} mismatches between content file name and file contents.");
return(mismatchedContentHashCount == 0 && mismatchedParentDirectoryCount == 0);
}
private Task <Result <LogFile> > WriteLogsToFileAsync(OperationContext context, AbsolutePath logFilePath, string[] logs)
{
return(context.PerformOperationAsync(Tracer, async() =>
{
using var fileStream = await _fileSystem.OpenSafeAsync(
logFilePath,
FileAccess.Write,
FileMode.CreateNew,
FileShare.None,
FileOptions.SequentialScan | FileOptions.Asynchronous);
using var gzipStream = new GZipStream(fileStream, CompressionLevel.Fastest, leaveOpen: true);
using var recordingStream = new CountingStream(gzipStream);
using var streamWriter = new StreamWriter(recordingStream, Encoding.UTF8, bufferSize: 32 * 1024, leaveOpen: true);
if (OnFileOpen != null)
{
await OnFileOpen(streamWriter);
}
foreach (var log in logs)
{
await streamWriter.WriteLineAsync(log);
}
if (OnFileClose != null)
{
await OnFileClose(streamWriter);
}
await streamWriter.FlushAsync();
Tracer.TrackMetric(context, $"LogLinesWritten", logs.Length);
var compressedSizeBytes = fileStream.Position;
Tracer.TrackMetric(context, $"CompressedBytesWritten", compressedSizeBytes);
var uncompressedSizeBytes = recordingStream.BytesWritten;
Tracer.TrackMetric(context, $"UncompressedBytesWritten", uncompressedSizeBytes);
return new Result <LogFile>(new LogFile()
{
Path = logFilePath,
UncompressedSizeBytes = uncompressedSizeBytes,
CompressedSizeBytes = compressedSizeBytes,
});
},
/// <summary>
/// Serialize a ContentStoreConfiguration to JSON in the standard filename in a CAS root directory.
/// </summary>
public static async Task Write(this ContentStoreConfiguration configuration, IAbsFileSystem fileSystem, AbsolutePath rootPath)
{
Contract.Requires(fileSystem != null);
Contract.Requires(rootPath != null);
Contract.Requires(configuration != null);
Contract.Requires(configuration.IsValid);
AbsolutePath jsonPath = rootPath / FileName;
if (!fileSystem.DirectoryExists(rootPath))
{
throw new CacheException($"Directory path=[{rootPath}] does not exist");
}
using (var stream =
await fileSystem.OpenSafeAsync(jsonPath, FileAccess.Write, FileMode.Create, FileShare.None))
{
configuration.SerializeToJSON(stream);
}
}
/// <summary>
/// Serialize hibernated session information to the standard filename in the given directory.
/// </summary>
public static async Task WriteAsync(this HibernatedSessions sessions, IAbsFileSystem fileSystem, AbsolutePath rootPath)
{
Contract.Requires(fileSystem != null);
Contract.Requires(rootPath != null);
// Due to abnormal process termination, the file that we'll be writing can be corrupted.
// To prevent this issue we first write the file into a temporary location and then we "move it" into a final location.
using (var tempFolder = new DisposableDirectory(fileSystem, rootPath / "Temp"))
{
var jsonTempPath = tempFolder.CreateRandomFileName();
var jsonPath = rootPath / FileName;
using (var stream =
await fileSystem.OpenSafeAsync(jsonTempPath, FileAccess.Write, FileMode.Create, FileShare.None))
{
sessions.SerializeToJSON(stream);
}
fileSystem.MoveFile(jsonTempPath, jsonPath, replaceExisting: true);
}
}
/// <summary>
/// AcquireAsync the lock, waiting as long as it takes or until the configured timeout.
/// </summary>
public async Task <LockAcquisitionResult> AcquireAsync(Context context, TimeSpan waitTimeout)
{
_tracer.Info(context, $"Acquiring lock file=[{_lockFilePath}]");
_fileSystem.CreateDirectory(_lockFilePath.GetParent());
DateTime timeOutTime = DateTime.UtcNow + waitTimeout;
Exception?lastException = null;
int? lastCompetingProcessId = null;
while (DateTime.UtcNow < timeOutTime)
{
try
{
// Anything other than FileShare.None is effectively ignored in Unix
FileShare fileShare = BuildXL.Utilities.OperatingSystemHelper.IsUnixOS ? FileShare.None : FileShare.Read;
_lockFile = await _fileSystem.OpenSafeAsync(
_lockFilePath, FileAccess.Write, FileMode.OpenOrCreate, fileShare);
using (var writer = new StreamWriter(_lockFile, UTF8WithoutBom, bufferSize: 4096, leaveOpen: true))
{
await writer.WriteLineAsync(
$"Lock acquired at {DateTime.UtcNow:O} by computer [{Environment.MachineName}] running command line [{Environment.CommandLine}] with process id [{Process.GetCurrentProcess().Id}]"
);
}
_tracer.Info(context, $"Acquired lock file=[{_lockFilePath}]");
await _lockFile.FlushAsync();
return(LockAcquisitionResult.Acquired());
}
catch (IOException ioException)
{
lastException = ioException;
}
catch (UnauthorizedAccessException accessException)
{
lastException = accessException;
}
try
{
string?contents = await _fileSystem.TryReadFileAsync(_lockFilePath);
if (contents != null)
{
_tracer.Diagnostic(context, $"Lock file=[{_lockFilePath}] contains [{contents}]");
lastCompetingProcessId = TryExtractProcessIdFromLockFilesContent(contents);
}
}
catch (Exception readLockFileException)
{
string message = readLockFileException is UnauthorizedAccessException ae ? ae.Message : readLockFileException.ToString();
// This is just extra cautious. We shouldn't fail hard being unable to get this diagnostic information.
_tracer.Info(
context,
$"Unable to read contents of lock file=[{_lockFilePath}] because [{message}]");
}
await Task.Delay(_pollingInterval);
}
string lastProcessIdText = lastCompetingProcessId == null ? string.Empty : " Competing process Id: " + lastCompetingProcessId;
_tracer.Info(
context,
$"Timed out trying to acquire lock file=[{_lockFilePath}].{lastProcessIdText} Last exception was=[{lastException}]");
return(LockAcquisitionResult.Failed(waitTimeout, lastCompetingProcessId, TryGetProcessName(lastCompetingProcessId), lastException));
}
public async Task HandlePushFileAsync(IAsyncStreamReader <PushFileRequest> requestStream, IServerStreamWriter <PushFileResponse> responseStream, ServerCallContext callContext)
{
// Detaching from the calling thread to (potentially) avoid IO Completion port thread exhaustion
await Task.Yield();
var startTime = DateTime.UtcNow;
var pushRequest = PushRequest.FromMetadata(callContext.RequestHeaders);
var hash = pushRequest.Hash;
var cacheContext = new Context(pushRequest.TraceId, Logger);
// Cancelling the operation when the instance is shut down.
using var shutdownTracker = TrackShutdown(cacheContext, callContext.CancellationToken);
var token = shutdownTracker.Context.Token;
var store = _contentStoreByCacheName.Values.OfType <IPushFileHandler>().FirstOrDefault();
var rejection = CanHandlePushRequest(cacheContext, hash, store);
if (rejection != RejectionReason.Accepted)
{
await callContext.WriteResponseHeadersAsync(PushResponse.DoNotCopy(rejection).Metadata);
return;
}
try
{
// Running the logic inside try/finally block to remove the hash being processed regardless of the result of this method.
await callContext.WriteResponseHeadersAsync(PushResponse.Copy.Metadata);
PutResult?result = null;
using (var disposableFile = new DisposableFile(cacheContext, _fileSystem, _temporaryDirectory !.CreateRandomFileName()))
{
// NOTE(jubayard): DeleteOnClose not used here because the file needs to be placed into the CAS.
// Opening a file for read/write and then doing pretty much anything to it leads to weird behavior
// that needs to be tested on a case by case basis. Since we don't know what the underlying store
// plans to do with the file, it is more robust to just use the DisposableFile construct.
using (var tempFile = await _fileSystem.OpenSafeAsync(disposableFile.Path, FileAccess.Write, FileMode.CreateNew, FileShare.None))
{
// From the docs: On the server side, MoveNext() does not throw exceptions.
// In case of a failure, the request stream will appear to be finished (MoveNext will return false)
// and the CancellationToken associated with the call will be cancelled to signal the failure.
while (await requestStream.MoveNext(token))
{
var request = requestStream.Current;
var bytes = request.Content.ToByteArray();
await tempFile.Stream.WriteAsync(bytes, 0, bytes.Length, token);
}
}
if (token.IsCancellationRequested)
{
if (!callContext.CancellationToken.IsCancellationRequested)
{
await responseStream.WriteAsync(new PushFileResponse()
{
Header = ResponseHeader.Failure(startTime, "Operation cancelled by handler.")
});
}
var cancellationSource = callContext.CancellationToken.IsCancellationRequested ? "caller" : "handler";
cacheContext.Debug($"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} cancelled by {cancellationSource}.");
return;
}
result = await store.HandlePushFileAsync(cacheContext, hash, disposableFile.Path, token);
}
var response = result
? new PushFileResponse {
Header = ResponseHeader.Success(startTime)
}
: new PushFileResponse {
Header = ResponseHeader.Failure(startTime, result.ErrorMessage)
};
await responseStream.WriteAsync(response);
}
finally
{
lock (_pushesLock)
{
_ongoingPushes.Remove(hash);
}
}
}