private void InitializeIncrementalCheckpointIfNeeded() { // If incremental checkpoint info is not initialized. Clean up incremental checkpoint directory // before proceeding if (_configuration.UseIncrementalCheckpointing) { if (_incrementalCheckpointInfo.Count == 0) { _fileSystem.CreateDirectory(_incrementalCheckpointDirectory); if (_fileSystem.FileExists(_incrementalCheckpointInfoFile)) { // An incremental checkpoint exists. Make sure that it is loaded _incrementalCheckpointInfo = ParseCheckpointInfo(_incrementalCheckpointInfoFile); } } // Synchronize incremental checkpoint directory with incremental checkpoint file var files = _fileSystem.EnumerateFiles(_incrementalCheckpointDirectory, EnumerateOptions.Recurse).Select(s => s.FullPath).ToList(); foreach (var file in files) { if (file != _incrementalCheckpointInfoFile) { var relativePath = file.Path.Substring(_incrementalCheckpointDirectory.Path.Length + 1); if (!_incrementalCheckpointInfo.ContainsKey(relativePath)) { _fileSystem.DeleteFile(file); } } } } }
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { await _container.CreateIfNotExistsAsync( accessType : BlobContainerPublicAccessType.Off, options : null, operationContext : null, cancellationToken : context.Token); // Any logs in the staging are basically lost: they were in memory only, and we crashed or failed as we // were writing them. We just recreate the directory. try { _fileSystem.DeleteDirectory(_configuration.StagingFolderPath, DeleteOptions.Recurse); } catch (DirectoryNotFoundException) { } _fileSystem.CreateDirectory(_configuration.StagingFolderPath); _fileSystem.CreateDirectory(_configuration.UploadFolderPath); _writeQueue.Start(WriteBatchAsync); _uploadQueue.Start(UploadBatchAsync); return(RecoverFromCrash(context)); }
private void WithLogManager(TimeSpan retention, Action <OperationContext, RocksDbLogsManager> action) { var backupFolder = _workingDirectory.Path / "backup"; _fileSystem.CreateDirectory(backupFolder); Assert.True(_fileSystem.DirectoryExists(backupFolder)); var tracingContext = new Context(TestGlobal.Logger); var operationContext = new OperationContext(tracingContext); var manager = new RocksDbLogsManager(_clock, _fileSystem, backupFolder, retention); action(operationContext, manager); }
/// <summary> /// Creates an empty file at a given path. /// </summary> public static async Task CreateEmptyFileAsync(this IAbsFileSystem fileSystem, AbsolutePath path) { fileSystem.CreateDirectory(path.Parent); using (await fileSystem.OpenAsync(path, FileAccess.Write, FileMode.Create, FileShare.None, FileOptions.None, bufferSize: 1).ConfigureAwait(false)) { } }
/// <inheritdoc /> protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { if (!FileSystem.DirectoryExists(Config.DataRootPath)) { FileSystem.CreateDirectory(Config.DataRootPath); } await StartupStoresAsync(context).ThrowIfFailure(); await LoadHibernatedSessionsAsync(context); InitializeAndStartGrpcServer(Config.GrpcPort, BindServices(), Config.RequestCallTokensPerCompletionQueue); _serviceReadinessChecker.Ready(context); _sessionExpirationCheckTimer = new IntervalTimer( () => CheckForExpiredSessionsAsync(context), MinTimeSpan(Config.UnusedSessionHeartbeatTimeout, TimeSpan.FromMinutes(CheckForExpiredSessionsPeriodMinutes)), message => Tracer.Debug(context, $"[{CheckForExpiredSessionsName}] message")); _logIncrementalStatsTimer = new IntervalTimer( () => LogIncrementalStatsAsync(context), Config.LogIncrementalStatsInterval); return(BoolResult.Success); }
/// <nodoc /> public LocalCacheServer( IAbsFileSystem fileSystem, ILogger logger, string scenario, Func <AbsolutePath, ICache> cacheFactory, LocalServerConfiguration localContentServerConfiguration, Capabilities capabilities = Capabilities.All) : base(logger, fileSystem, scenario, cacheFactory, localContentServerConfiguration) { var storesByName = new Dictionary <string, IContentStore>(); foreach (var kvp in localContentServerConfiguration.NamedCacheRoots) { AbsolutePath cacheRootPath = kvp.Value; fileSystem.CreateDirectory(cacheRootPath); var cache = cacheFactory(cacheRootPath); Contract.Assert(cache is IContentStore, $"Attempted to setup a cache named '{kvp.Key}' that is not an {nameof(IContentStore)} at path {cacheRootPath}, type used is {cache.GetType().Name}"); storesByName.Add(kvp.Key, (IContentStore)cache); } _grpcContentServer = new GrpcContentServer(logger, capabilities, this, storesByName); _grpcCacheServer = new GrpcCacheServer(logger, this); }
/// <summary> /// Initializes a new instance of the <see cref="DisposableDirectory" /> class. /// </summary> public DisposableDirectory(IAbsFileSystem fileSystem, AbsolutePath directoryPath) { Contract.Requires(fileSystem != null); _fileSystem = fileSystem; Path = directoryPath; fileSystem.CreateDirectory(directoryPath); }
/// <summary> /// Load value from the filesystem. /// </summary> public static Guid Load(IAbsFileSystem fileSystem, AbsolutePath filePath) { Guid guid; try { // First try reading the GUID file guid = Read(fileSystem, filePath); } catch (Exception e) when(e is CacheException || e is IOException) { // If that fails, we likely need to create a Guid guid = CacheDeterminism.NewCacheGuid(); try { fileSystem.CreateDirectory(filePath.Parent); // Write the Guid file fileSystem.WriteAllBytes(filePath, Encoding.UTF8.GetBytes(guid.ToString(SerializationFormat))); } catch (Exception ex) when(ex is IOException) { // If we failed to write the Guid file we may have just missed getting the guid in the first place, // so let us try to read it again. This failure we let go all the way out. guid = Read(fileSystem, filePath); } } return(guid); }
private static void WriteContentStoreConfigFile(string cacheSizeQuotaString, AbsolutePath rootPath, IAbsFileSystem fileSystem) { fileSystem.CreateDirectory(rootPath); var maxSizeQuota = new MaxSizeQuota(cacheSizeQuotaString); var casConfig = new ContentStoreConfiguration(maxSizeQuota); casConfig.Write(fileSystem, rootPath).GetAwaiter().GetResult(); }
/// <inheritdoc /> protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { // Splitting initialization into two pieces: // Normal startup procedure and post-initialization step that notifies all // the special stores that the initialization has finished. // This is a workaround to make sure hibernated sessions are fully restored // before FileSystemContentStore can evict the content. var result = await tryStartupCoreAsync(); foreach (var store in StoresByName.Values) { if (store is IContentStore contentStore) { contentStore.PostInitializationCompleted(context, result); } } return(result); async Task <BoolResult> tryStartupCoreAsync() { try { if (!FileSystem.DirectoryExists(Config.DataRootPath)) { FileSystem.CreateDirectory(Config.DataRootPath); } await StartupStoresAsync(context).ThrowIfFailure(); await LoadHibernatedSessionsAsync(context); InitializeAndStartGrpcServer(Config.GrpcPort, BindServices(), Config.RequestCallTokensPerCompletionQueue, Config.GrpcThreadPoolSize ?? DefaultGrpcThreadPoolSize); _serviceReadinessChecker.Ready(context); _sessionExpirationCheckTimer = new IntervalTimer( () => CheckForExpiredSessionsAsync(context), MinTimeSpan(Config.UnusedSessionHeartbeatTimeout, TimeSpan.FromMinutes(CheckForExpiredSessionsPeriodMinutes)), message => Tracer.Debug(context, $"[{CheckForExpiredSessionsName}] message")); _logIncrementalStatsTimer = new IntervalTimer( () => LogIncrementalStatsAsync(context), Config.LogIncrementalStatsInterval); _logMachineStatsTimer = new IntervalTimer( () => LogMachinePerformanceStatistics(context), Config.LogMachineStatsInterval); return(BoolResult.Success); } catch (Exception e) { return(new BoolResult(e)); } } }
/// <summary> /// Creates an empty file at a given path. /// </summary> public static void CreateEmptyFile(this IAbsFileSystem fileSystem, AbsolutePath path) { Contract.RequiresNotNull(path); Contract.RequiresNotNull(path.Parent); fileSystem.CreateDirectory(path.Parent); using (fileSystem.TryOpen(path, FileAccess.Write, FileMode.Create, FileShare.None, FileOptions.None, bufferSize: 1)) { } }
/// <summary> /// Initializes a new instance of the <see cref="SerializedDataValue" /> class. /// </summary> /// <param name="fileSystem">File system to use.</param> /// <param name="valueFilePath">Directory to track.</param> /// <param name="initialValue">Value to set if the directory does not initially exist.</param> public SerializedDataValue(IAbsFileSystem fileSystem, AbsolutePath valueFilePath, int initialValue) { Contract.Requires(fileSystem != null); Contract.Requires(valueFilePath != null); _fileSystem = fileSystem; _valueFilePath = valueFilePath; if (!_fileSystem.FileExists(_valueFilePath)) { _fileSystem.CreateDirectory(_valueFilePath.GetParent()); WriteValueFile(initialValue); } }
public async Task <Result <AbsolutePath> > BackupAsync(OperationContext context, AbsolutePath instancePath, string?name = null) { int numCopiedFiles = 0; AbsolutePath?backupPath = null; return(await context.PerformOperationAsync(_tracer, async() => { var backupTime = _clock.UtcNow.ToString("yyyyMMdd_HHmmss", CultureInfo.InvariantCulture); var backupName = backupTime; if (!string.IsNullOrEmpty(name)) { backupName += $"-{name}"; } backupPath = _backupPath / backupName; // Unlikely, but it is possible for GC to start running and think that it should purge this directory, // this avoids the scenario. using var _ = await _locks.AcquireAsync(backupPath); if (_fileSystem.DirectoryExists(backupPath)) { _fileSystem.DeleteDirectory(backupPath, DeleteOptions.All); } _fileSystem.CreateDirectory(backupPath); // See: https://github.com/facebook/rocksdb/wiki/rocksdb-basics#database-debug-logs _fileSystem.EnumerateFiles(instancePath, "*LOG*", false, fileInfo => { var fileName = fileInfo.FullPath.FileName; var targetFilePath = backupPath / fileName; // Do not use Async here: since EnumerateFiles takes an Action, making this async means we'll // need to make the action async as well, which is equivalent to an async void function. That // leads to race conditions. #pragma warning disable AsyncFixer02 // Long running or blocking operations under an async method _fileSystem.CopyFile(fileInfo.FullPath, targetFilePath, replaceExisting: true); #pragma warning restore AsyncFixer02 // Long running or blocking operations under an async method ++numCopiedFiles; }); if (numCopiedFiles == 0) { _fileSystem.DeleteDirectory(backupPath, DeleteOptions.All); } return new Result <AbsolutePath>(backupPath); }, extraEndMessage : _ => $"From=[{instancePath}] To=[{backupPath?.ToString() ?? "Unknown"}] NumCopiedFiles=[{numCopiedFiles}]"));
/// <nodoc /> protected LocalContentServerBase( ILogger logger, IAbsFileSystem fileSystem, string scenario, Func <AbsolutePath, TStore> contentStoreFactory, LocalServerConfiguration localContentServerConfiguration, IGrpcServiceEndpoint[]?additionalEndpoints) { Contract.Requires(logger != null); Contract.Requires(fileSystem != null); Contract.Requires(localContentServerConfiguration != null); Contract.Requires(localContentServerConfiguration.GrpcPort > 0, "GrpcPort must be provided"); logger.Debug($"{Name} process id {Process.GetCurrentProcess().Id}"); logger.Debug($"{Name} constructing {nameof(ServiceConfiguration)}: {localContentServerConfiguration}"); GrpcEnvironment.Initialize(logger, localContentServerConfiguration.GrpcEnvironmentOptions, overwriteSafeOptions: true); FileSystem = fileSystem; Logger = logger; Config = localContentServerConfiguration; _additionalEndpoints = additionalEndpoints ?? Array.Empty <IGrpcServiceEndpoint>(); _serviceReadinessChecker = new ServiceReadinessChecker(logger, scenario); _sessionHandles = new ConcurrentDictionary <int, ISessionHandle <TSession, TSessionData> >(); var storesByName = new Dictionary <string, TStore>(); foreach (var kvp in localContentServerConfiguration.NamedCacheRoots) { fileSystem.CreateDirectory(kvp.Value); var store = contentStoreFactory(kvp.Value); storesByName.Add(kvp.Key, store); } StoresByName = new ReadOnlyDictionary <string, TStore>(storesByName); foreach (var kvp in localContentServerConfiguration.NamedCacheRoots) { _tempFolderForStreamsByCacheName[kvp.Key] = kvp.Value / "TempFolder"; } if (!string.IsNullOrEmpty(localContentServerConfiguration.GrpcPortFileName)) { var portSharingFactory = new MemoryMappedFileGrpcPortSharingFactory(logger, localContentServerConfiguration.GrpcPortFileName); var portExposer = portSharingFactory.GetPortExposer(); _portDisposer = portExposer.Expose(localContentServerConfiguration.GrpcPort); } }
public async Task <Result <AbsolutePath> > BackupAsync(OperationContext context, AbsolutePath instancePath, string?name = null) { int numCopiedFiles = 0; AbsolutePath?backupPath = null; return(await context.PerformOperationAsync(_tracer, async() => { var backupTime = _clock.UtcNow.ToString("yyyyMMdd_HHmmss", CultureInfo.InvariantCulture); var backupName = backupTime; if (!string.IsNullOrEmpty(name)) { backupName += $"-{name}"; } backupPath = _backupPath / backupName; // Unlikely, but it is possible for GC to start running and think that it should purge this directory, // this avoids the scenario. using var _ = await _locks.AcquireAsync(backupPath); if (_fileSystem.DirectoryExists(backupPath)) { _fileSystem.DeleteDirectory(backupPath, DeleteOptions.All); } _fileSystem.CreateDirectory(backupPath); // See: https://github.com/facebook/rocksdb/wiki/rocksdb-basics#database-debug-logs _fileSystem.EnumerateFiles(instancePath, "*LOG*", false, async fileInfo => { var fileName = fileInfo.FullPath.FileName; var targetFilePath = backupPath / fileName; await _fileSystem.CopyFileAsync(fileInfo.FullPath, targetFilePath, replaceExisting: true); ++numCopiedFiles; }); if (numCopiedFiles == 0) { _fileSystem.DeleteDirectory(backupPath, DeleteOptions.All); } return new Result <AbsolutePath>(backupPath); }, extraEndMessage : _ => $"From=[{instancePath}] To=[{backupPath?.ToString() ?? "Unknown"}] NumCopiedFiles=[{numCopiedFiles}]"));
/// <inheritdoc /> public CheckpointManager( ContentLocationDatabase database, ICheckpointRegistry checkpointRegistry, CentralStorage storage, CheckpointConfiguration configuration, CounterCollection <ContentLocationStoreCounters> counters) { _database = database; _checkpointRegistry = checkpointRegistry; _storage = storage; _configuration = configuration; _fileSystem = new PassThroughFileSystem(); _checkpointStagingDirectory = configuration.WorkingDirectory / "staging"; _incrementalCheckpointDirectory = configuration.WorkingDirectory / "incremental"; _fileSystem.CreateDirectory(_incrementalCheckpointDirectory); _incrementalCheckpointInfoFile = _incrementalCheckpointDirectory / "checkpointInfo.txt"; Counters = counters; }
/// <nodoc /> protected LocalContentServerBase( ILogger logger, IAbsFileSystem fileSystem, string scenario, Func <AbsolutePath, TStore> contentStoreFactory, LocalServerConfiguration localContentServerConfiguration) { Contract.Requires(logger != null); Contract.Requires(fileSystem != null); Contract.Requires(localContentServerConfiguration != null); Contract.Requires(localContentServerConfiguration.GrpcPort > 0, "GrpcPort must be provided"); logger.Debug($"{Name} process id {Process.GetCurrentProcess().Id}"); logger.Debug($"{Name} constructing {nameof(ServiceConfiguration)}: {localContentServerConfiguration}"); FileSystem = fileSystem; Logger = logger; Config = localContentServerConfiguration; _serviceReadinessChecker = new ServiceReadinessChecker(Tracer, logger, scenario); _sessionHandles = new ConcurrentDictionary <int, SessionHandle <TSession> >(); foreach (var kvp in localContentServerConfiguration.NamedCacheRoots) { fileSystem.CreateDirectory(kvp.Value); var store = contentStoreFactory(kvp.Value); StoresByName.Add(kvp.Key, store); } foreach (var kvp in localContentServerConfiguration.NamedCacheRoots) { _tempFolderForStreamsByCacheName[kvp.Key] = kvp.Value / "TempFolder"; } if (!string.IsNullOrEmpty(localContentServerConfiguration.GrpcPortFileName)) { var portSharingFactory = new MemoryMappedFileGrpcPortSharingFactory(logger, localContentServerConfiguration.GrpcPortFileName); var portExposer = portSharingFactory.GetPortExposer(); _portDisposer = portExposer.Expose(localContentServerConfiguration.GrpcPort); } }
protected override async Task <BoolResult> StartupCoreAsync(OperationContext context) { // Splitting initialization into two pieces: // Normal startup procedure and post-initialization step that notifies all // the special stores that the initialization has finished. // This is a workaround to make sure hibernated sessions are fully restored // before FileSystemContentStore can evict the content. var result = await tryStartupCoreAsync(); if (!result) { // We should not be running post initialization operation if the startup operation failed. return(result); } foreach (var store in StoresByName.Values) { if (store is IContentStore contentStore) { contentStore.PostInitializationCompleted(context, result); } } return(result); async Task <BoolResult> tryStartupCoreAsync() { try { if (!FileSystem.DirectoryExists(Config.DataRootPath)) { FileSystem.CreateDirectory(Config.DataRootPath); } await StartupStoresAsync(context).ThrowIfFailure(); foreach (var endpoint in GrpcEndpoints) { await endpoint.StartupAsync(context).ThrowIfFailure(); } await LoadHibernatedSessionsAsync(context); if (!Config.DisableGrpcServer) { InitializeAndStartGrpcServer(context, Config); } _serviceReadinessChecker.Ready(context); _sessionExpirationCheckTimer = new IntervalTimer( () => CheckForExpiredSessionsAsync(context), MinTimeSpan(Config.UnusedSessionHeartbeatTimeout, TimeSpan.FromMinutes(CheckForExpiredSessionsPeriodMinutes)), logAction: message => Tracer.Debug(context, $"{CheckForExpiredSessionsName}: {message}")); _logIncrementalStatsTimer = new IntervalTimer( () => LogIncrementalStatsAsync(context, logAtShutdown: false), Config.LogIncrementalStatsInterval); _logMachineStatsTimer = new IntervalTimer( () => LogMachinePerformanceStatistics(context), Config.LogMachineStatsInterval); return(BoolResult.Success); } catch (Exception e) { return(new BoolResult(e)); } } }
/// <summary> /// AcquireAsync the lock, waiting as long as it takes or until the configured timeout. /// </summary> public async Task <LockAcquisitionResult> AcquireAsync(Context context, TimeSpan waitTimeout) { _tracer.Info(context, $"Acquiring lock file=[{_lockFilePath}]"); _fileSystem.CreateDirectory(_lockFilePath.GetParent()); DateTime timeOutTime = DateTime.UtcNow + waitTimeout; Exception?lastException = null; int? lastCompetingProcessId = null; while (DateTime.UtcNow < timeOutTime) { try { // Anything other than FileShare.None is effectively ignored in Unix FileShare fileShare = BuildXL.Utilities.OperatingSystemHelper.IsUnixOS ? FileShare.None : FileShare.Read; _lockFile = await _fileSystem.OpenSafeAsync( _lockFilePath, FileAccess.Write, FileMode.OpenOrCreate, fileShare); using (var writer = new StreamWriter(_lockFile, UTF8WithoutBom, bufferSize: 4096, leaveOpen: true)) { await writer.WriteLineAsync( $"Lock acquired at {DateTime.UtcNow:O} by computer [{Environment.MachineName}] running command line [{Environment.CommandLine}] with process id [{Process.GetCurrentProcess().Id}]" ); } _tracer.Info(context, $"Acquired lock file=[{_lockFilePath}]"); await _lockFile.FlushAsync(); return(LockAcquisitionResult.Acquired()); } catch (IOException ioException) { lastException = ioException; } catch (UnauthorizedAccessException accessException) { lastException = accessException; } try { string?contents = await _fileSystem.TryReadFileAsync(_lockFilePath); if (contents != null) { _tracer.Diagnostic(context, $"Lock file=[{_lockFilePath}] contains [{contents}]"); lastCompetingProcessId = TryExtractProcessIdFromLockFilesContent(contents); } } catch (Exception readLockFileException) { string message = readLockFileException is UnauthorizedAccessException ae ? ae.Message : readLockFileException.ToString(); // This is just extra cautious. We shouldn't fail hard being unable to get this diagnostic information. _tracer.Info( context, $"Unable to read contents of lock file=[{_lockFilePath}] because [{message}]"); } await Task.Delay(_pollingInterval); } string lastProcessIdText = lastCompetingProcessId == null ? string.Empty : " Competing process Id: " + lastCompetingProcessId; _tracer.Info( context, $"Timed out trying to acquire lock file=[{_lockFilePath}].{lastProcessIdText} Last exception was=[{lastException}]"); return(LockAcquisitionResult.Failed(waitTimeout, lastCompetingProcessId, TryGetProcessName(lastCompetingProcessId), lastException)); }