private void InitializeIncrementalCheckpointIfNeeded()
{
// If incremental checkpoint info is not initialized. Clean up incremental checkpoint directory
// before proceeding
if (_configuration.UseIncrementalCheckpointing)
{
if (_incrementalCheckpointInfo.Count == 0)
{
_fileSystem.CreateDirectory(_incrementalCheckpointDirectory);
if (_fileSystem.FileExists(_incrementalCheckpointInfoFile))
{
// An incremental checkpoint exists. Make sure that it is loaded
_incrementalCheckpointInfo = ParseCheckpointInfo(_incrementalCheckpointInfoFile);
}
}
// Synchronize incremental checkpoint directory with incremental checkpoint file
var files = _fileSystem.EnumerateFiles(_incrementalCheckpointDirectory, EnumerateOptions.Recurse).Select(s => s.FullPath).ToList();
foreach (var file in files)
{
if (file != _incrementalCheckpointInfoFile)
{
var relativePath = file.Path.Substring(_incrementalCheckpointDirectory.Path.Length + 1);
if (!_incrementalCheckpointInfo.ContainsKey(relativePath))
{
_fileSystem.DeleteFile(file);
}
}
}
}
}
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
await _container.CreateIfNotExistsAsync(
accessType : BlobContainerPublicAccessType.Off,
options : null,
operationContext : null,
cancellationToken : context.Token);
// Any logs in the staging are basically lost: they were in memory only, and we crashed or failed as we
// were writing them. We just recreate the directory.
try
{
_fileSystem.DeleteDirectory(_configuration.StagingFolderPath, DeleteOptions.Recurse);
}
catch (DirectoryNotFoundException)
{
}
_fileSystem.CreateDirectory(_configuration.StagingFolderPath);
_fileSystem.CreateDirectory(_configuration.UploadFolderPath);
_writeQueue.Start(WriteBatchAsync);
_uploadQueue.Start(UploadBatchAsync);
return(RecoverFromCrash(context));
}
private void WithLogManager(TimeSpan retention, Action <OperationContext, RocksDbLogsManager> action)
{
var backupFolder = _workingDirectory.Path / "backup";
_fileSystem.CreateDirectory(backupFolder);
Assert.True(_fileSystem.DirectoryExists(backupFolder));
var tracingContext = new Context(TestGlobal.Logger);
var operationContext = new OperationContext(tracingContext);
var manager = new RocksDbLogsManager(_clock, _fileSystem, backupFolder, retention);
action(operationContext, manager);
}
/// <summary>
/// Creates an empty file at a given path.
/// </summary>
public static async Task CreateEmptyFileAsync(this IAbsFileSystem fileSystem, AbsolutePath path)
{
fileSystem.CreateDirectory(path.Parent);
using (await fileSystem.OpenAsync(path, FileAccess.Write, FileMode.Create, FileShare.None, FileOptions.None, bufferSize: 1).ConfigureAwait(false))
{
}
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
if (!FileSystem.DirectoryExists(Config.DataRootPath))
{
FileSystem.CreateDirectory(Config.DataRootPath);
}
await StartupStoresAsync(context).ThrowIfFailure();
await LoadHibernatedSessionsAsync(context);
InitializeAndStartGrpcServer(Config.GrpcPort, BindServices(), Config.RequestCallTokensPerCompletionQueue);
_serviceReadinessChecker.Ready(context);
_sessionExpirationCheckTimer = new IntervalTimer(
() => CheckForExpiredSessionsAsync(context),
MinTimeSpan(Config.UnusedSessionHeartbeatTimeout, TimeSpan.FromMinutes(CheckForExpiredSessionsPeriodMinutes)),
message => Tracer.Debug(context, $"[{CheckForExpiredSessionsName}] message"));
_logIncrementalStatsTimer = new IntervalTimer(
() => LogIncrementalStatsAsync(context),
Config.LogIncrementalStatsInterval);
return(BoolResult.Success);
}
/// <nodoc />
public LocalCacheServer(
IAbsFileSystem fileSystem,
ILogger logger,
string scenario,
Func <AbsolutePath, ICache> cacheFactory,
LocalServerConfiguration localContentServerConfiguration,
Capabilities capabilities = Capabilities.All)
: base(logger, fileSystem, scenario, cacheFactory, localContentServerConfiguration)
{
var storesByName = new Dictionary <string, IContentStore>();
foreach (var kvp in localContentServerConfiguration.NamedCacheRoots)
{
AbsolutePath cacheRootPath = kvp.Value;
fileSystem.CreateDirectory(cacheRootPath);
var cache = cacheFactory(cacheRootPath);
Contract.Assert(cache is IContentStore, $"Attempted to setup a cache named '{kvp.Key}' that is not an {nameof(IContentStore)} at path {cacheRootPath}, type used is {cache.GetType().Name}");
storesByName.Add(kvp.Key, (IContentStore)cache);
}
_grpcContentServer = new GrpcContentServer(logger, capabilities, this, storesByName);
_grpcCacheServer = new GrpcCacheServer(logger, this);
}
/// <summary>
/// Initializes a new instance of the <see cref="DisposableDirectory" /> class.
/// </summary>
public DisposableDirectory(IAbsFileSystem fileSystem, AbsolutePath directoryPath)
{
Contract.Requires(fileSystem != null);
_fileSystem = fileSystem;
Path = directoryPath;
fileSystem.CreateDirectory(directoryPath);
}
/// <summary>
/// Load value from the filesystem.
/// </summary>
public static Guid Load(IAbsFileSystem fileSystem, AbsolutePath filePath)
{
Guid guid;
try
{
// First try reading the GUID file
guid = Read(fileSystem, filePath);
}
catch (Exception e) when(e is CacheException || e is IOException)
{
// If that fails, we likely need to create a Guid
guid = CacheDeterminism.NewCacheGuid();
try
{
fileSystem.CreateDirectory(filePath.Parent);
// Write the Guid file
fileSystem.WriteAllBytes(filePath, Encoding.UTF8.GetBytes(guid.ToString(SerializationFormat)));
}
catch (Exception ex) when(ex is IOException)
{
// If we failed to write the Guid file we may have just missed getting the guid in the first place,
// so let us try to read it again. This failure we let go all the way out.
guid = Read(fileSystem, filePath);
}
}
return(guid);
}
private static void WriteContentStoreConfigFile(string cacheSizeQuotaString, AbsolutePath rootPath, IAbsFileSystem fileSystem)
{
fileSystem.CreateDirectory(rootPath);
var maxSizeQuota = new MaxSizeQuota(cacheSizeQuotaString);
var casConfig = new ContentStoreConfiguration(maxSizeQuota);
casConfig.Write(fileSystem, rootPath).GetAwaiter().GetResult();
}
/// <inheritdoc />
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
// Splitting initialization into two pieces:
// Normal startup procedure and post-initialization step that notifies all
// the special stores that the initialization has finished.
// This is a workaround to make sure hibernated sessions are fully restored
// before FileSystemContentStore can evict the content.
var result = await tryStartupCoreAsync();
foreach (var store in StoresByName.Values)
{
if (store is IContentStore contentStore)
{
contentStore.PostInitializationCompleted(context, result);
}
}
return(result);
async Task <BoolResult> tryStartupCoreAsync()
{
try
{
if (!FileSystem.DirectoryExists(Config.DataRootPath))
{
FileSystem.CreateDirectory(Config.DataRootPath);
}
await StartupStoresAsync(context).ThrowIfFailure();
await LoadHibernatedSessionsAsync(context);
InitializeAndStartGrpcServer(Config.GrpcPort, BindServices(), Config.RequestCallTokensPerCompletionQueue, Config.GrpcThreadPoolSize ?? DefaultGrpcThreadPoolSize);
_serviceReadinessChecker.Ready(context);
_sessionExpirationCheckTimer = new IntervalTimer(
() => CheckForExpiredSessionsAsync(context),
MinTimeSpan(Config.UnusedSessionHeartbeatTimeout, TimeSpan.FromMinutes(CheckForExpiredSessionsPeriodMinutes)),
message => Tracer.Debug(context, $"[{CheckForExpiredSessionsName}] message"));
_logIncrementalStatsTimer = new IntervalTimer(
() => LogIncrementalStatsAsync(context),
Config.LogIncrementalStatsInterval);
_logMachineStatsTimer = new IntervalTimer(
() => LogMachinePerformanceStatistics(context),
Config.LogMachineStatsInterval);
return(BoolResult.Success);
}
catch (Exception e)
{
return(new BoolResult(e));
}
}
}
/// <summary>
/// Creates an empty file at a given path.
/// </summary>
public static void CreateEmptyFile(this IAbsFileSystem fileSystem, AbsolutePath path)
{
Contract.RequiresNotNull(path);
Contract.RequiresNotNull(path.Parent);
fileSystem.CreateDirectory(path.Parent);
using (fileSystem.TryOpen(path, FileAccess.Write, FileMode.Create, FileShare.None, FileOptions.None, bufferSize: 1))
{
}
}
/// <summary>
/// Initializes a new instance of the <see cref="SerializedDataValue" /> class.
/// </summary>
/// <param name="fileSystem">File system to use.</param>
/// <param name="valueFilePath">Directory to track.</param>
/// <param name="initialValue">Value to set if the directory does not initially exist.</param>
public SerializedDataValue(IAbsFileSystem fileSystem, AbsolutePath valueFilePath, int initialValue)
{
Contract.Requires(fileSystem != null);
Contract.Requires(valueFilePath != null);
_fileSystem = fileSystem;
_valueFilePath = valueFilePath;
if (!_fileSystem.FileExists(_valueFilePath))
{
_fileSystem.CreateDirectory(_valueFilePath.GetParent());
WriteValueFile(initialValue);
}
}
public async Task <Result <AbsolutePath> > BackupAsync(OperationContext context, AbsolutePath instancePath, string?name = null)
{
int numCopiedFiles = 0;
AbsolutePath?backupPath = null;
return(await context.PerformOperationAsync(_tracer, async() =>
{
var backupTime = _clock.UtcNow.ToString("yyyyMMdd_HHmmss", CultureInfo.InvariantCulture);
var backupName = backupTime;
if (!string.IsNullOrEmpty(name))
{
backupName += $"-{name}";
}
backupPath = _backupPath / backupName;
// Unlikely, but it is possible for GC to start running and think that it should purge this directory,
// this avoids the scenario.
using var _ = await _locks.AcquireAsync(backupPath);
if (_fileSystem.DirectoryExists(backupPath))
{
_fileSystem.DeleteDirectory(backupPath, DeleteOptions.All);
}
_fileSystem.CreateDirectory(backupPath);
// See: https://github.com/facebook/rocksdb/wiki/rocksdb-basics#database-debug-logs
_fileSystem.EnumerateFiles(instancePath, "*LOG*", false,
fileInfo =>
{
var fileName = fileInfo.FullPath.FileName;
var targetFilePath = backupPath / fileName;
// Do not use Async here: since EnumerateFiles takes an Action, making this async means we'll
// need to make the action async as well, which is equivalent to an async void function. That
// leads to race conditions.
#pragma warning disable AsyncFixer02 // Long running or blocking operations under an async method
_fileSystem.CopyFile(fileInfo.FullPath, targetFilePath, replaceExisting: true);
#pragma warning restore AsyncFixer02 // Long running or blocking operations under an async method
++numCopiedFiles;
});
if (numCopiedFiles == 0)
{
_fileSystem.DeleteDirectory(backupPath, DeleteOptions.All);
}
return new Result <AbsolutePath>(backupPath);
}, extraEndMessage : _ => $"From=[{instancePath}] To=[{backupPath?.ToString() ?? "Unknown"}] NumCopiedFiles=[{numCopiedFiles}]"));
/// <nodoc />
protected LocalContentServerBase(
ILogger logger,
IAbsFileSystem fileSystem,
string scenario,
Func <AbsolutePath, TStore> contentStoreFactory,
LocalServerConfiguration localContentServerConfiguration,
IGrpcServiceEndpoint[]?additionalEndpoints)
{
Contract.Requires(logger != null);
Contract.Requires(fileSystem != null);
Contract.Requires(localContentServerConfiguration != null);
Contract.Requires(localContentServerConfiguration.GrpcPort > 0, "GrpcPort must be provided");
logger.Debug($"{Name} process id {Process.GetCurrentProcess().Id}");
logger.Debug($"{Name} constructing {nameof(ServiceConfiguration)}: {localContentServerConfiguration}");
GrpcEnvironment.Initialize(logger, localContentServerConfiguration.GrpcEnvironmentOptions, overwriteSafeOptions: true);
FileSystem = fileSystem;
Logger = logger;
Config = localContentServerConfiguration;
_additionalEndpoints = additionalEndpoints ?? Array.Empty <IGrpcServiceEndpoint>();
_serviceReadinessChecker = new ServiceReadinessChecker(logger, scenario);
_sessionHandles = new ConcurrentDictionary <int, ISessionHandle <TSession, TSessionData> >();
var storesByName = new Dictionary <string, TStore>();
foreach (var kvp in localContentServerConfiguration.NamedCacheRoots)
{
fileSystem.CreateDirectory(kvp.Value);
var store = contentStoreFactory(kvp.Value);
storesByName.Add(kvp.Key, store);
}
StoresByName = new ReadOnlyDictionary <string, TStore>(storesByName);
foreach (var kvp in localContentServerConfiguration.NamedCacheRoots)
{
_tempFolderForStreamsByCacheName[kvp.Key] = kvp.Value / "TempFolder";
}
if (!string.IsNullOrEmpty(localContentServerConfiguration.GrpcPortFileName))
{
var portSharingFactory = new MemoryMappedFileGrpcPortSharingFactory(logger, localContentServerConfiguration.GrpcPortFileName);
var portExposer = portSharingFactory.GetPortExposer();
_portDisposer = portExposer.Expose(localContentServerConfiguration.GrpcPort);
}
}
public async Task <Result <AbsolutePath> > BackupAsync(OperationContext context, AbsolutePath instancePath, string?name = null)
{
int numCopiedFiles = 0;
AbsolutePath?backupPath = null;
return(await context.PerformOperationAsync(_tracer, async() =>
{
var backupTime = _clock.UtcNow.ToString("yyyyMMdd_HHmmss", CultureInfo.InvariantCulture);
var backupName = backupTime;
if (!string.IsNullOrEmpty(name))
{
backupName += $"-{name}";
}
backupPath = _backupPath / backupName;
// Unlikely, but it is possible for GC to start running and think that it should purge this directory,
// this avoids the scenario.
using var _ = await _locks.AcquireAsync(backupPath);
if (_fileSystem.DirectoryExists(backupPath))
{
_fileSystem.DeleteDirectory(backupPath, DeleteOptions.All);
}
_fileSystem.CreateDirectory(backupPath);
// See: https://github.com/facebook/rocksdb/wiki/rocksdb-basics#database-debug-logs
_fileSystem.EnumerateFiles(instancePath, "*LOG*", false,
async fileInfo =>
{
var fileName = fileInfo.FullPath.FileName;
var targetFilePath = backupPath / fileName;
await _fileSystem.CopyFileAsync(fileInfo.FullPath, targetFilePath, replaceExisting: true);
++numCopiedFiles;
});
if (numCopiedFiles == 0)
{
_fileSystem.DeleteDirectory(backupPath, DeleteOptions.All);
}
return new Result <AbsolutePath>(backupPath);
}, extraEndMessage : _ => $"From=[{instancePath}] To=[{backupPath?.ToString() ?? "Unknown"}] NumCopiedFiles=[{numCopiedFiles}]"));
/// <inheritdoc />
public CheckpointManager(
ContentLocationDatabase database,
ICheckpointRegistry checkpointRegistry,
CentralStorage storage,
CheckpointConfiguration configuration,
CounterCollection <ContentLocationStoreCounters> counters)
{
_database = database;
_checkpointRegistry = checkpointRegistry;
_storage = storage;
_configuration = configuration;
_fileSystem = new PassThroughFileSystem();
_checkpointStagingDirectory = configuration.WorkingDirectory / "staging";
_incrementalCheckpointDirectory = configuration.WorkingDirectory / "incremental";
_fileSystem.CreateDirectory(_incrementalCheckpointDirectory);
_incrementalCheckpointInfoFile = _incrementalCheckpointDirectory / "checkpointInfo.txt";
Counters = counters;
}
/// <nodoc />
protected LocalContentServerBase(
ILogger logger,
IAbsFileSystem fileSystem,
string scenario,
Func <AbsolutePath, TStore> contentStoreFactory,
LocalServerConfiguration localContentServerConfiguration)
{
Contract.Requires(logger != null);
Contract.Requires(fileSystem != null);
Contract.Requires(localContentServerConfiguration != null);
Contract.Requires(localContentServerConfiguration.GrpcPort > 0, "GrpcPort must be provided");
logger.Debug($"{Name} process id {Process.GetCurrentProcess().Id}");
logger.Debug($"{Name} constructing {nameof(ServiceConfiguration)}: {localContentServerConfiguration}");
FileSystem = fileSystem;
Logger = logger;
Config = localContentServerConfiguration;
_serviceReadinessChecker = new ServiceReadinessChecker(Tracer, logger, scenario);
_sessionHandles = new ConcurrentDictionary <int, SessionHandle <TSession> >();
foreach (var kvp in localContentServerConfiguration.NamedCacheRoots)
{
fileSystem.CreateDirectory(kvp.Value);
var store = contentStoreFactory(kvp.Value);
StoresByName.Add(kvp.Key, store);
}
foreach (var kvp in localContentServerConfiguration.NamedCacheRoots)
{
_tempFolderForStreamsByCacheName[kvp.Key] = kvp.Value / "TempFolder";
}
if (!string.IsNullOrEmpty(localContentServerConfiguration.GrpcPortFileName))
{
var portSharingFactory = new MemoryMappedFileGrpcPortSharingFactory(logger, localContentServerConfiguration.GrpcPortFileName);
var portExposer = portSharingFactory.GetPortExposer();
_portDisposer = portExposer.Expose(localContentServerConfiguration.GrpcPort);
}
}
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context)
{
// Splitting initialization into two pieces:
// Normal startup procedure and post-initialization step that notifies all
// the special stores that the initialization has finished.
// This is a workaround to make sure hibernated sessions are fully restored
// before FileSystemContentStore can evict the content.
var result = await tryStartupCoreAsync();
if (!result)
{
// We should not be running post initialization operation if the startup operation failed.
return(result);
}
foreach (var store in StoresByName.Values)
{
if (store is IContentStore contentStore)
{
contentStore.PostInitializationCompleted(context, result);
}
}
return(result);
async Task <BoolResult> tryStartupCoreAsync()
{
try
{
if (!FileSystem.DirectoryExists(Config.DataRootPath))
{
FileSystem.CreateDirectory(Config.DataRootPath);
}
await StartupStoresAsync(context).ThrowIfFailure();
foreach (var endpoint in GrpcEndpoints)
{
await endpoint.StartupAsync(context).ThrowIfFailure();
}
await LoadHibernatedSessionsAsync(context);
if (!Config.DisableGrpcServer)
{
InitializeAndStartGrpcServer(context, Config);
}
_serviceReadinessChecker.Ready(context);
_sessionExpirationCheckTimer = new IntervalTimer(
() => CheckForExpiredSessionsAsync(context),
MinTimeSpan(Config.UnusedSessionHeartbeatTimeout, TimeSpan.FromMinutes(CheckForExpiredSessionsPeriodMinutes)),
logAction: message => Tracer.Debug(context, $"{CheckForExpiredSessionsName}: {message}"));
_logIncrementalStatsTimer = new IntervalTimer(
() => LogIncrementalStatsAsync(context, logAtShutdown: false),
Config.LogIncrementalStatsInterval);
_logMachineStatsTimer = new IntervalTimer(
() => LogMachinePerformanceStatistics(context),
Config.LogMachineStatsInterval);
return(BoolResult.Success);
}
catch (Exception e)
{
return(new BoolResult(e));
}
}
}
/// <summary>
/// AcquireAsync the lock, waiting as long as it takes or until the configured timeout.
/// </summary>
public async Task <LockAcquisitionResult> AcquireAsync(Context context, TimeSpan waitTimeout)
{
_tracer.Info(context, $"Acquiring lock file=[{_lockFilePath}]");
_fileSystem.CreateDirectory(_lockFilePath.GetParent());
DateTime timeOutTime = DateTime.UtcNow + waitTimeout;
Exception?lastException = null;
int? lastCompetingProcessId = null;
while (DateTime.UtcNow < timeOutTime)
{
try
{
// Anything other than FileShare.None is effectively ignored in Unix
FileShare fileShare = BuildXL.Utilities.OperatingSystemHelper.IsUnixOS ? FileShare.None : FileShare.Read;
_lockFile = await _fileSystem.OpenSafeAsync(
_lockFilePath, FileAccess.Write, FileMode.OpenOrCreate, fileShare);
using (var writer = new StreamWriter(_lockFile, UTF8WithoutBom, bufferSize: 4096, leaveOpen: true))
{
await writer.WriteLineAsync(
$"Lock acquired at {DateTime.UtcNow:O} by computer [{Environment.MachineName}] running command line [{Environment.CommandLine}] with process id [{Process.GetCurrentProcess().Id}]"
);
}
_tracer.Info(context, $"Acquired lock file=[{_lockFilePath}]");
await _lockFile.FlushAsync();
return(LockAcquisitionResult.Acquired());
}
catch (IOException ioException)
{
lastException = ioException;
}
catch (UnauthorizedAccessException accessException)
{
lastException = accessException;
}
try
{
string?contents = await _fileSystem.TryReadFileAsync(_lockFilePath);
if (contents != null)
{
_tracer.Diagnostic(context, $"Lock file=[{_lockFilePath}] contains [{contents}]");
lastCompetingProcessId = TryExtractProcessIdFromLockFilesContent(contents);
}
}
catch (Exception readLockFileException)
{
string message = readLockFileException is UnauthorizedAccessException ae ? ae.Message : readLockFileException.ToString();
// This is just extra cautious. We shouldn't fail hard being unable to get this diagnostic information.
_tracer.Info(
context,
$"Unable to read contents of lock file=[{_lockFilePath}] because [{message}]");
}
await Task.Delay(_pollingInterval);
}
string lastProcessIdText = lastCompetingProcessId == null ? string.Empty : " Competing process Id: " + lastCompetingProcessId;
_tracer.Info(
context,
$"Timed out trying to acquire lock file=[{_lockFilePath}].{lastProcessIdText} Last exception was=[{lastException}]");
return(LockAcquisitionResult.Failed(waitTimeout, lastCompetingProcessId, TryGetProcessName(lastCompetingProcessId), lastException));
}