private LocalCache( ILogger logger, AbsolutePath rootPath, IAbsFileSystem fileSystem, IClock clock, ConfigurationModel configurationModel, SQLiteMemoizationStoreConfiguration memoConfig, ContentStoreSettings contentStoreSettings, LocalCacheConfiguration localCacheConfiguration) : base( () => localCacheConfiguration.EnableContentServer ? (IContentStore) new ServiceClientContentStore( logger, fileSystem, localCacheConfiguration.CacheName, new ServiceClientRpcConfiguration(localCacheConfiguration.GrpcPort), (uint)localCacheConfiguration.RetryIntervalSeconds, (uint)localCacheConfiguration.RetryCount, scenario: localCacheConfiguration.ScenarioName) : new FileSystemContentStore( fileSystem, clock, rootPath, configurationModel: configurationModel, settings: contentStoreSettings), () => new SQLiteMemoizationStore( logger, clock ?? SystemClock.Instance, memoConfig), PersistentId.Load(fileSystem, rootPath / IdFileName)) { _fileSystem = fileSystem; }
private static MemoizationStoreConfiguration GetMemoizationStoreConfiguration(AbsolutePath cacheRoot, Config config, CasConfig configCore) { if (config.UseRocksDbMemoizationStore) { return(new RocksDbMemoizationStoreConfiguration() { Database = new RocksDbContentLocationDatabaseConfiguration(cacheRoot / "RocksDbMemoizationStore") { CleanOnInitialize = false, }, }); } else { var memoConfig = new SQLiteMemoizationStoreConfiguration(cacheRoot) { MaxRowCount = config.MaxStrongFingerprints, SingleInstanceTimeoutSeconds = (int)configCore.SingleInstanceTimeoutInSeconds, WaitForLruOnShutdown = WaitForLruOnShutdown }; memoConfig.Database.BackupDatabase = config.BackupLKGCache; memoConfig.Database.VerifyIntegrityOnStartup = config.CheckCacheIntegrityOnStartup; if (!string.IsNullOrEmpty(config.SynchronizationMode)) { memoConfig.Database.SyncMode = (SynchronizationMode)Enum.Parse(typeof(SynchronizationMode), config.SynchronizationMode, ignoreCase: true); } return(memoConfig); } }
private static MemoizationStoreConfiguration CreateInProcMemoizationStoreConfiguration(Config cacheConfig, AbsolutePath cacheRootPath) { if (cacheConfig.UseRoxisMetadataStore) { var roxisClientConfiguration = new Roxis.Client.RoxisClientConfiguration(); if (!string.IsNullOrEmpty(cacheConfig.RoxisMetadataStoreHost)) { roxisClientConfiguration.GrpcHost = cacheConfig.RoxisMetadataStoreHost; } if (cacheConfig.RoxisMetadataStorePort > 0) { roxisClientConfiguration.GrpcPort = cacheConfig.RoxisMetadataStorePort; } return(new RoxisMemoizationDatabaseConfiguration() { MetadataClientConfiguration = roxisClientConfiguration, }); } if (cacheConfig.UseRocksDbMemoizationStore) { return(new RocksDbMemoizationStoreConfiguration() { Database = new RocksDbContentLocationDatabaseConfiguration(cacheRootPath / "RocksDbMemoizationStore") { CleanOnInitialize = false, GarbageCollectionInterval = TimeSpan.FromSeconds(cacheConfig.RocksDbMemoizationStoreGarbageCollectionIntervalInSeconds), MetadataGarbageCollectionEnabled = true, MetadataGarbageCollectionMaximumNumberOfEntriesToKeep = cacheConfig.RocksDbMemoizationStoreGarbageCollectionMaximumNumberOfEntriesToKeep, OnFailureDeleteExistingStoreAndRetry = true, LogsKeepLongTerm = true, }, }); } else { var memoConfig = new SQLiteMemoizationStoreConfiguration(cacheRootPath) { MaxRowCount = cacheConfig.MaxStrongFingerprints, SingleInstanceTimeoutSeconds = (int)cacheConfig.SingleInstanceTimeoutInSeconds }; memoConfig.Database.BackupDatabase = cacheConfig.BackupLKGCache; memoConfig.Database.VerifyIntegrityOnStartup = cacheConfig.CheckCacheIntegrityOnStartup; if (!string.IsNullOrEmpty(cacheConfig.SynchronizationMode)) { memoConfig.Database.SyncMode = (SynchronizationMode)Enum.Parse(typeof(SynchronizationMode), cacheConfig.SynchronizationMode, ignoreCase: true); } return(memoConfig); } }
protected override IMemoizationStore CreateStore(DisposableDirectory testDirectory) { var memoConfig = new SQLiteMemoizationStoreConfiguration(testDirectory.Path) { MaxRowCount = MaxRowCount }; memoConfig.Database.JournalMode = ContentStore.SQLite.JournalMode.OFF; return(new SQLiteMemoizationStore(Logger, _clock, memoConfig)); }
public TestSQLiteMemoizationStore( ILogger logger, IClock clock, SQLiteMemoizationStoreConfiguration config) : base( logger, clock, config) { }
private IMemoizationStore CreateSQLiteMemoizationStore(AbsolutePath path, SynchronizationMode syncMode = SQLiteMemoizationStore.DefaultSyncMode) { var memoConfig = new SQLiteMemoizationStoreConfiguration(path) { MaxRowCount = MaxRowCount }; memoConfig.Database.SyncMode = syncMode; memoConfig.Database.JournalMode = JournalMode.OFF; return(new TestSQLiteMemoizationStore(Logger, _clock, memoConfig)); }
private Task RunTestWithIntegrityCheckAtStartupAsync(Context context, DisposableDirectory testDirectory, System.Func <IMemoizationStore, IMemoizationSession, Task> funcAsync) { return(RunTestAsync(context, testDirectory, funcAsync, _ => { var memoConfig = new SQLiteMemoizationStoreConfiguration(testDirectory.Path) { MaxRowCount = MaxRowCount }; memoConfig.Database.VerifyIntegrityOnStartup = true; return new TestSQLiteMemoizationStore(Logger, _clock, memoConfig); })); }
private static SQLiteMemoizationStoreConfiguration GenerateMemoizationConfiguration(long maxRowCount, SynchronizationMode syncMode, DisposableDirectory testDirectory) { var memoConfig = new SQLiteMemoizationStoreConfiguration(testDirectory.Path) { MaxRowCount = maxRowCount }; memoConfig.Database.SyncMode = syncMode; // Having the journal disabled won't be indicative of real world performance. It is disabled so the tests run faster memoConfig.Database.JournalMode = JournalMode.OFF; return(memoConfig); }
protected override ICache CreateCache(DisposableDirectory testDirectory) { var rootPath = testDirectory.Path; var configuration = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1); configuration.Write(FileSystem, rootPath).Wait(); var memoConfig = new SQLiteMemoizationStoreConfiguration(rootPath) { MaxRowCount = MaxContentHashListItems }; memoConfig.Database.JournalMode = ContentStore.SQLite.JournalMode.OFF; return(LocalCache.CreateUnknownContentStoreInProcMemoizationStoreCache(Logger, rootPath, memoConfig, LocalCacheConfiguration.CreateServerDisabled(), clock: Clock)); }
private LocalCache( ILogger logger, AbsolutePath rootPath, IAbsFileSystem fileSystem, IClock clock, ServiceClientContentStoreConfiguration configuration, SQLiteMemoizationStoreConfiguration memoConfig) : base( () => new ServiceClientContentStore( logger, fileSystem, configuration), () => new SQLiteMemoizationStore( logger, clock ?? SystemClock.Instance, memoConfig), PersistentId.Load(fileSystem, rootPath / IdFileName)) { _fileSystem = fileSystem; }
private void RunSQLiteStoreSession( AbsolutePath rootPath, bool lruEnabled, System.Func <Context, SQLiteMemoizationStore, SQLiteMemoizationSession, Task> funcAsync) { var config = new SQLiteMemoizationStoreConfiguration(rootPath) { MaxRowCount = lruEnabled ? _maxRowCount : 0, WaitForLruOnShutdown = _waitForLruOnShutdown, }; System.Func <SQLiteMemoizationStore> createFunc = () => new SQLiteMemoizationStore( _logger, SystemClock.Instance, config); RunMemoizationStore(createFunc, funcAsync); }
private static SQLiteMemoizationStoreConfiguration GetMemoConfig(AbsolutePath cacheRoot, Config config, CasConfig configCore) { var memoConfig = new SQLiteMemoizationStoreConfiguration(cacheRoot) { MaxRowCount = config.MaxStrongFingerprints, BackupDatabase = config.BackupLKGCache, VerifyIntegrityOnStartup = config.CheckCacheIntegrityOnStartup, SingleInstanceTimeoutSeconds = (int)configCore.SingleInstanceTimeoutInSeconds, WaitForLruOnShutdown = WaitForLruOnShutdown }; if (!string.IsNullOrEmpty(config.SynchronizationMode)) { memoConfig.SyncMode = (SynchronizationMode)Enum.Parse(typeof(SynchronizationMode), config.SynchronizationMode, ignoreCase: true); } return(memoConfig); }
/// <summary> /// Initializes a new instance of the <see cref="LocalCache" /> class backed by <see cref="FileSystemContentStore"/> /// </summary> public LocalCache( ILogger logger, AbsolutePath rootPath, SQLiteMemoizationStoreConfiguration memoConfig, ConfigurationModel configurationModel = null, IClock clock = null, bool checkLocalFiles = true, bool emptyFileHashShortcutEnabled = false) : this( logger, rootPath, new PassThroughFileSystem(logger), clock ?? SystemClock.Instance, configurationModel, memoConfig, checkLocalFiles, emptyFileHashShortcutEnabled) { }
protected override ICache CreateCache(DisposableDirectory testDirectory) { var rootPathForStream = RootPathOfContentStoreForStream(testDirectory); var rootPathForPath = RootPathOfContentStoreForPath(testDirectory); FileSystem.CreateDirectory(rootPathForStream); FileSystem.CreateDirectory(rootPathForPath); var configuration1 = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1); var configuration2 = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1); configuration1.Write(FileSystem, rootPathForStream).Wait(); configuration2.Write(FileSystem, rootPathForPath).Wait(); var memoConfig = new SQLiteMemoizationStoreConfiguration(rootPathForPath) { MaxRowCount = MaxContentHashListItems }; memoConfig.Database.JournalMode = ContentStore.SQLite.JournalMode.OFF; return(LocalCache.CreateStreamPathContentStoreInProcMemoizationStoreCache(Logger, rootPathForStream, rootPathForPath, memoConfig, clock: Clock)); }
private LocalCache( ILogger logger, AbsolutePath rootPathForStream, AbsolutePath rootPathForPath, IAbsFileSystem fileSystem, IClock clock, ConfigurationModel configurationModelForStream, ConfigurationModel configurationModelForPath, SQLiteMemoizationStoreConfiguration memoConfig, bool checkLocalFiles, bool emptyFileHashShortcutEnabled) : base( () => new StreamPathContentStore( () => new FileSystemContentStore(fileSystem, clock ?? SystemClock.Instance, rootPathForStream, configurationModelForStream, settings: new ContentStoreSettings() { CheckFiles = checkLocalFiles, UseEmptyFileHashShortcut = emptyFileHashShortcutEnabled }), () => new FileSystemContentStore(fileSystem, clock ?? SystemClock.Instance, rootPathForPath, configurationModelForPath, settings: new ContentStoreSettings() { CheckFiles = checkLocalFiles, UseEmptyFileHashShortcut = emptyFileHashShortcutEnabled })), () => new SQLiteMemoizationStore( logger, clock ?? SystemClock.Instance, memoConfig), PersistentId.Load(fileSystem, rootPathForPath / IdFileName)) { _fileSystem = fileSystem; }
/// <summary> /// Initializes a new instance of the <see cref="LocalCache" /> class backed by <see cref="ServiceClientContentStore"/> /// </summary> public LocalCache( ILogger logger, string cacheName, AbsolutePath rootPath, ServiceClientRpcConfiguration rpcConfiguration, uint retryIntervalSeconds, uint retryCount, SQLiteMemoizationStoreConfiguration memoConfig, IClock clock = null, string scenarioName = null) : this( logger, rootPath, new PassThroughFileSystem(logger), clock ?? SystemClock.Instance, new ServiceClientContentStoreConfiguration(cacheName, rpcConfiguration, scenarioName) { RetryCount = retryCount, RetryIntervalSeconds = retryIntervalSeconds, }, memoConfig) { }
private static MemoizationStoreConfiguration GetInProcMemoizationStoreConfiguration(AbsolutePath cacheRoot, Config config, CasConfig configCore) { if (config.UseRocksDbMemoizationStore) { return(new RocksDbMemoizationStoreConfiguration() { Database = new RocksDbContentLocationDatabaseConfiguration(cacheRoot / "RocksDbMemoizationStore") { CleanOnInitialize = false, GarbageCollectionInterval = TimeSpan.FromSeconds(config.RocksDbMemoizationStoreGarbageCollectionIntervalInSeconds), MetadataGarbageCollectionEnabled = true, MetadataGarbageCollectionMaximumNumberOfEntriesToKeep = config.RocksDbMemoizationStoreGarbageCollectionMaximumNumberOfEntriesToKeep, OnFailureDeleteExistingStoreAndRetry = true, LogsKeepLongTerm = false, }, }); } else { var memoConfig = new SQLiteMemoizationStoreConfiguration(cacheRoot) { MaxRowCount = config.MaxStrongFingerprints, SingleInstanceTimeoutSeconds = (int)configCore.SingleInstanceTimeoutInSeconds, WaitForLruOnShutdown = WaitForLruOnShutdown }; memoConfig.Database.BackupDatabase = config.BackupLKGCache; memoConfig.Database.VerifyIntegrityOnStartup = config.CheckCacheIntegrityOnStartup; if (!string.IsNullOrEmpty(config.SynchronizationMode)) { memoConfig.Database.SyncMode = (SynchronizationMode)Enum.Parse(typeof(SynchronizationMode), config.SynchronizationMode, ignoreCase: true); } return(memoConfig); } }
private static MemoizationStoreConfiguration CreateMemoizationStoreConfiguration(Config config, AbsolutePath cacheRootPath) { if (config.UseRocksDbMemoizationStore) { var memoConfig = new RocksDbMemoizationStoreConfiguration() { Database = new RocksDbContentLocationDatabaseConfiguration(cacheRootPath / "RocksDbMemoizationStore") { CleanOnInitialize = false, GarbageCollectionInterval = TimeSpan.FromSeconds(config.RocksDbMemoizationStoreGarbageCollectionIntervalInSeconds), MetadataGarbageCollectionEnabled = true, MetadataGarbageCollectionProtectionTime = TimeSpan.FromSeconds(config.RocksDbMemoizationStoreGarbageCollectionProtectionTimeInSeconds), }, }; return(memoConfig); } else { var memoConfig = new SQLiteMemoizationStoreConfiguration(cacheRootPath) { MaxRowCount = config.MaxStrongFingerprints, SingleInstanceTimeoutSeconds = (int)config.SingleInstanceTimeoutInSeconds }; memoConfig.Database.BackupDatabase = config.BackupLKGCache; memoConfig.Database.VerifyIntegrityOnStartup = config.CheckCacheIntegrityOnStartup; if (!string.IsNullOrEmpty(config.SynchronizationMode)) { memoConfig.Database.SyncMode = (SynchronizationMode)Enum.Parse(typeof(SynchronizationMode), config.SynchronizationMode, ignoreCase: true); } return(memoConfig); } }
/// <inheritdoc /> public async Task <Possible <ICache, Failure> > InitializeCacheAsync(ICacheConfigData cacheData, Guid activityId) { Contract.Requires(cacheData != null); var possibleCacheConfig = cacheData.Create <Config>(); if (!possibleCacheConfig.Succeeded) { return(possibleCacheConfig.Failure); } Config cacheConfig = possibleCacheConfig.Result; try { Contract.Assert(cacheConfig.CacheName != null); var logPath = new AbsolutePath(cacheConfig.MetadataLogPath); var logger = new DisposeLogger(() => new EtwFileLog(logPath.Path, cacheConfig.CacheId), cacheConfig.LogFlushIntervalSeconds); logger.Debug($"Creating CASaaS backed LocalCache using cache name: {cacheConfig.CacheName}"); ServiceClientRpcConfiguration rpcConfiguration; if (cacheConfig.GrpcPort != 0) { rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort); } else { var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName); var portReader = factory.GetPortReader(); var port = portReader.ReadPort(); rpcConfiguration = new ServiceClientRpcConfiguration(port); } var rootPath = new AbsolutePath(cacheConfig.MetadataRootPath); var memoConfig = new SQLiteMemoizationStoreConfiguration(rootPath) { MaxRowCount = cacheConfig.MaxStrongFingerprints, BackupDatabase = cacheConfig.BackupLKGCache, VerifyIntegrityOnStartup = cacheConfig.CheckCacheIntegrityOnStartup, SingleInstanceTimeoutSeconds = (int)cacheConfig.SingleInstanceTimeoutInSeconds }; if (!string.IsNullOrEmpty(cacheConfig.SynchronizationMode)) { memoConfig.SyncMode = (SynchronizationMode)Enum.Parse(typeof(SynchronizationMode), cacheConfig.SynchronizationMode, ignoreCase: true); } var localCache = new LocalCache( logger, cacheConfig.CacheName, rootPath, rpcConfiguration, cacheConfig.ConnectionRetryIntervalSeconds, cacheConfig.ConnectionRetryCount, memoConfig, scenarioName: cacheConfig.ScenarioName); var statsFilePath = new AbsolutePath(logPath.Path + ".stats"); var cache = new MemoizationStoreAdapterCache(cacheConfig.CacheId, localCache, logger, statsFilePath, cacheConfig.ReplaceExistingOnPlaceFile); var startupResult = await cache.StartupAsync(); if (!startupResult.Succeeded) { logger.Error($"Error while initializing the cache [{cacheConfig.CacheId}]. Failure: {startupResult.Failure}"); return(startupResult.Failure); } logger.Debug("Successfully started CloudStoreLocalCacheService client."); return(cache); } catch (Exception e) { return(new CacheConstructionFailure(cacheConfig.CacheId, e)); } }