/// <summary> /// Returns a cache initializer to a real instance of the cache /// </summary> protected CacheInitializer GetRealCacheInitializerForTests() { var tempDir = OperatingSystemHelper.IsUnixOS ? "/tmp/buildxl-temp" : TemporaryDirectory; string cacheDirectory = Path.Combine(tempDir, "cache"); AbsolutePath cacheConfigPath = WriteTestCacheConfigToDisk(cacheDirectory); var translator = new RootTranslator(); if (TryGetSubstSourceAndTarget(out var substSource, out var substTarget)) { translator.AddTranslation(substTarget, substSource); } translator.Seal(); Configuration.Cache.CacheConfigFile = cacheConfigPath; Configuration.Cache.CacheLogFilePath = AbsolutePath.Create(Context.PathTable, tempDir).Combine(Context.PathTable, "cache.log"); var maybeCacheInitializer = CacheInitializer.GetCacheInitializationTask( LoggingContext, Context.PathTable, cacheDirectory, Configuration.Cache, translator, recoveryStatus: false, cancellationToken: CancellationToken.None).GetAwaiter().GetResult(); if (!maybeCacheInitializer.Succeeded) { throw new BuildXLException("Unable to initialize the real cache: " + maybeCacheInitializer.Failure.DescribeIncludingInnerFailures()); } return(maybeCacheInitializer.Result); }
public void RootTranslation() { var rootTranslator = new RootTranslator(); var pathTable = Context.PathTable; // Source is shorter than target var shortCaseSourceTranslatedRootPath = GetFullPath(pathTable, "ShortSource").Expand(pathTable); var shortCaseTargetTranslatedRootPath = GetFullPath(pathTable, "Short___Target").Expand(pathTable); rootTranslator.AddTranslation( shortCaseSourceTranslatedRootPath.ExpandedPath, shortCaseTargetTranslatedRootPath.ExpandedPath); // Source is longer than target var longCaseSourceTranslatedRootPath = GetFullPath(pathTable, "LongSourceSource").Expand(pathTable); var longCaseTargetTranslatedRootPath = GetFullPath(pathTable, "LongTarget").Expand(pathTable); rootTranslator.AddTranslation( longCaseSourceTranslatedRootPath.ExpandedPath, longCaseTargetTranslatedRootPath.ExpandedPath); rootTranslator.Seal(); var cache = new CacheCoreArtifactContentCache(Session, rootTranslator); // SHORTER SOURCE, SAME ROOT: Path should NOT be translated VerifyExpandedPathForCacheEquals(cache, shortCaseSourceTranslatedRootPath, shortCaseSourceTranslatedRootPath); // LONGER SOURCE, SAME ROOT: Path SHOULD be translated VerifyExpandedPathForCacheEquals(cache, longCaseSourceTranslatedRootPath, longCaseTargetTranslatedRootPath); }
/// <nodoc /> public CacheCoreArtifactContentCache( ICacheSession cache, RootTranslator rootTranslator) { m_cache = new PossiblyOpenCacheSession(cache); m_rootTranslator = rootTranslator; }
/// <nodoc /> public CacheCoreArtifactContentCache( ICacheSession cache, RootTranslator rootTranslator, bool replaceExistingFileOnMaterialization = false) { m_cache = new PossiblyOpenCacheSession(cache); m_rootTranslator = rootTranslator; m_replaceExistingFileOnMaterialization = replaceExistingFileOnMaterialization; }
/// <summary> /// Gets an instance of <see cref="ICacheConfigData"/> from cache configuration. /// </summary> internal static Possible <ICacheConfigData> TryGetCacheConfigData( PathTable pathTable, string cacheDirectory, ICacheConfiguration config, RootTranslator rootTranslator = null) { Contract.Requires(pathTable != null); Contract.Requires(pathTable.IsValid); Contract.Requires(config != null); Contract.Requires(config.CacheLogFilePath.IsValid); Contract.Requires(config.CacheConfigFile.IsValid); Contract.Requires(!string.IsNullOrWhiteSpace(cacheDirectory)); Possible <string> maybeConfigData = TryReadCacheConfigFile(config.CacheConfigFile.ToString(pathTable)); if (!maybeConfigData.Succeeded) { return(maybeConfigData.Failure); } // Update the cache config to dynamically set the cache path if it is configured to use the per-invocation path. // TODO: Ideally this would be exposed as config constructor parameters to BuildXL to not require manipulating the json config. // But for now we just modify the config text before passing it along to the cache. string cacheConfigContent = maybeConfigData.Result; cacheConfigContent = cacheConfigContent.Replace("[DominoSelectedLogPath]", config.CacheLogFilePath.ToString(pathTable).Replace(@"\", @"\\")); // Escape path separation chars to json format cacheConfigContent = cacheConfigContent.Replace("[BuildXLSelectedLogPath]", config.CacheLogFilePath.ToString(pathTable).Replace(@"\", @"\\")); // Escape path separation chars to json format cacheConfigContent = cacheConfigContent.Replace("[DominoSelectedRootPath]", cacheDirectory.Replace(@"\", @"\\")); cacheConfigContent = cacheConfigContent.Replace("[BuildXLSelectedRootPath]", cacheDirectory.Replace(@"\", @"\\")); cacheConfigContent = cacheConfigContent.Replace("[UseDedupStore]", config.UseDedupStore.ToString()); cacheConfigContent = cacheConfigContent.Replace("[ReplaceExistingFileOnMaterialization]", config.ReplaceExistingFileOnMaterialization.ToString()); var vfsCasRoot = config.VfsCasRoot.IsValid ? config.VfsCasRoot.ToString(pathTable) : ""; if (rootTranslator != null && !string.IsNullOrEmpty(vfsCasRoot)) { // VFS needs real path so use root translator to resolve to real path. vfsCasRoot = rootTranslator.Translate(vfsCasRoot); } // Escape path separation chars to json format vfsCasRoot = vfsCasRoot.Replace(@"\", @"\\"); cacheConfigContent = cacheConfigContent.Replace("[VfsCasRoot]", vfsCasRoot); ICacheConfigData cacheConfigData; Exception exception; if (!CacheFactory.TryCreateCacheConfigData(cacheConfigContent, out cacheConfigData, out exception)) { return(new Failure <string>(I($"Unable to create cache config data: {exception.GetLogEventMessage()}"))); } return(new Possible <ICacheConfigData>(cacheConfigData)); }
private CacheCoreCacheInitializer( LoggingContext loggingContext, ICacheCoreCache cache, ICacheCoreSession session, List <IDisposable> acquiredDisposables, bool enableFingerprintLookup, RootTranslator rootTranslator) : base( loggingContext, acquiredDisposables, enableFingerprintLookup) { Contract.Requires(cache != null); Contract.Requires(session != null); m_cache = cache; m_session = session; m_rootTranslator = rootTranslator; m_initialStatistics = GetCacheBulkStatistics(session); }
public void RootTranslationDifferentRoots() { var rootTranslator = new RootTranslator(); var pathTable = Context.PathTable; // Source is shorter than target but has different root var shortCaseChangeRootSourceTranslatedRootPath = ChangeRoot(GetFullPath(pathTable, "ShortChangeRootSrc"), pathTable, newRoot: 'A'); var shortCaseChangeRootTargetTranslatedRootPath = ChangeRoot(GetFullPath(pathTable, "ShortChangeRootTarget"), pathTable, newRoot: 'B'); rootTranslator.AddTranslation( shortCaseChangeRootSourceTranslatedRootPath.ExpandedPath, shortCaseChangeRootTargetTranslatedRootPath.ExpandedPath); rootTranslator.Seal(); var cache = new CacheCoreArtifactContentCache(Session, rootTranslator); // SHORTER SOURCE, DIFFERENT ROOT: Path SHOULD be translated VerifyExpandedPathForCacheEquals(cache, shortCaseChangeRootSourceTranslatedRootPath, shortCaseChangeRootTargetTranslatedRootPath); }
public static CacheInitializationTask GetCacheInitializationTask( LoggingContext loggingContext, PathTable pathTable, string cacheDirectory, ICacheConfiguration config, RootTranslator rootTranslator, bool?recoveryStatus, CancellationToken cancellationToken, // Only used for testing purposes to inject cache. Func <EngineCache> testHookCacheFactory = null) { Contract.Requires(recoveryStatus.HasValue, "Recovery attempt should have been done before initializing the cache"); DateTime startTime = DateTime.UtcNow; var task = Task.Run( async() => { using (PerformanceMeasurement.Start( loggingContext, "CacheInitialization", Tracing.Logger.Log.StartInitializingCache, Tracing.Logger.Log.EndInitializingCache)) { if (testHookCacheFactory != null) { return(new MemoryCacheInitializer( testHookCacheFactory, loggingContext, new List <IDisposable>(), enableFingerprintLookup: config.Incremental)); } Possible <CacheCoreCacheInitializer> maybeCacheCoreEngineCache = await CacheCoreCacheInitializer.TryInitializeCacheInternalAsync( loggingContext, pathTable, cacheDirectory, config, enableFingerprintLookup: config.Incremental, rootTranslator: rootTranslator); if (!maybeCacheCoreEngineCache.Succeeded) { string errorMessage = maybeCacheCoreEngineCache.Failure.Describe(); if (errorMessage.Contains(LockAcquisitionFailureMessagePrefix)) { Tracing.Logger.Log.FailedToAcquireDirectoryLock( loggingContext, maybeCacheCoreEngineCache.Failure.DescribeIncludingInnerFailures()); } else { Tracing.Logger.Log.StorageCacheStartupError( loggingContext, maybeCacheCoreEngineCache.Failure.DescribeIncludingInnerFailures()); } } return(maybeCacheCoreEngineCache.Then <CacheInitializer>(c => c)); } }, cancellationToken); return(new CacheInitializationTask( loggingContext, startTime, task, cancellationToken)); }
internal static async Task <Possible <CacheCoreCacheInitializer> > TryInitializeCacheInternalAsync( LoggingContext loggingContext, PathTable pathTable, string cacheDirectory, ICacheConfiguration config, bool enableFingerprintLookup, RootTranslator rootTranslator) { Contract.Requires(pathTable != null); Contract.Requires(pathTable.IsValid); Contract.Requires(config != null); Contract.Requires(config.CacheLogFilePath.IsValid); Contract.Requires(config.CacheConfigFile.IsValid); Contract.Requires(!string.IsNullOrWhiteSpace(cacheDirectory)); bool succeeded = false; ICacheCoreCache cache = null; ICacheCoreSession session = null; try { Possible <ICacheConfigData> cacheConfigData = TryGetCacheConfigData(pathTable, cacheDirectory, config, rootTranslator); if (!cacheConfigData.Succeeded) { return(cacheConfigData.Failure); } Possible <ICacheCoreCache> maybeCache = await CacheFactory.InitializeCacheAsync(cacheConfigData.Result, loggingContext.ActivityId, config); if (!maybeCache.Succeeded) { return(maybeCache.Failure); } // We are now responsible for shutting this down (even if something later fails). cache = maybeCache.Result; cache.SuscribeForCacheStateDegredationFailures( failure => { Tracing.Logger.Log.CacheReportedRecoverableError(loggingContext, failure.DescribeIncludingInnerFailures()); }); // Log the cache ID we got. Tracing.Logger.Log.CacheInitialized(loggingContext, cache.CacheId); Possible <ICacheCoreSession> maybeSession = string.IsNullOrWhiteSpace(config.CacheSessionName) ? await cache.CreateSessionAsync() : await cache.CreateSessionAsync(config.CacheSessionName); if (!maybeSession.Succeeded) { return(maybeSession.Failure); } session = maybeSession.Result; succeeded = true; return(new CacheCoreCacheInitializer( loggingContext, cache, session, new List <IDisposable>(), enableFingerprintLookup: enableFingerprintLookup, rootTranslator: rootTranslator, replaceExistingFileOnMaterialization: config.ReplaceExistingFileOnMaterialization)); } finally { if (!succeeded) { // Note that we clean up in reverse order that we initialized things. if (session != null) { Analysis.IgnoreResult(await session.CloseAsync(), justification: "Okay to ignore close"); Analysis.IgnoreResult(await cache.ShutdownAsync(), justification: "Okay to ignore shutdown"); } } } }
public void TestProperLogMessageOnCacheLockAcquisitionFailure() { var tempDir = Path.Combine( OperatingSystemHelper.IsUnixOS ? "/tmp/bxl-temp" : TemporaryDirectory, Guid.NewGuid().ToString()); string cacheDirectory = Path.Combine(tempDir, "cache"); string cacheConfigJson = $@"{{ ""MaxCacheSizeInMB"": 1024, ""CacheId"": ""TestCache"", ""Assembly"": ""BuildXL.Cache.MemoizationStoreAdapter"", ""CacheLogPath"": ""[BuildXLSelectedLogPath]"", ""Type"": ""BuildXL.Cache.MemoizationStoreAdapter.MemoizationStoreCacheFactory"", ""CacheRootPath"": ""{cacheDirectory.Replace("\\", "\\\\")}"", ""UseStreamCAS"": false, ""SingleInstanceTimeoutInSeconds"" : 5 }}"; AbsolutePath cacheConfigPath = WriteTestCacheConfigToDisk(cacheDirectory, cacheConfigJson); var translator = new RootTranslator(); translator.Seal(); var possibleFirstCacheInitializer = CacheInitializer.GetCacheInitializationTask( LoggingContext, Context.PathTable, cacheDirectory, new CacheConfiguration { CacheLogFilePath = AbsolutePath.Create(Context.PathTable, tempDir).Combine(Context.PathTable, "cache.log"), CacheConfigFile = cacheConfigPath }, translator, recoveryStatus: false, cancellationToken: CancellationToken.None).GetAwaiter().GetResult(); if (!possibleFirstCacheInitializer.Succeeded) { AssertTrue(false, "Failed to initialize the cache: " + possibleFirstCacheInitializer.Failure.DescribeIncludingInnerFailures()); } var possibleSecondCacheInitializer = CacheInitializer.GetCacheInitializationTask( LoggingContext, Context.PathTable, cacheDirectory, new CacheConfiguration { // need a different name for the log file (due to the order in which the things are initialized) CacheLogFilePath = AbsolutePath.Create(Context.PathTable, tempDir).Combine(Context.PathTable, "cache_2.log"), CacheConfigFile = cacheConfigPath, }, translator, recoveryStatus: false, cancellationToken: CancellationToken.None).GetAwaiter().GetResult(); // close and dispose the first cache (must be done before the assert block bellow) var firstCacheInitializer = possibleFirstCacheInitializer.Result; AssertSuccess(firstCacheInitializer.Close()); firstCacheInitializer.Dispose(); AssertErrorEventLogged(global::BuildXL.Engine.Tracing.LogEventId.FailedToAcquireDirectoryLock); AssertTrue(!possibleSecondCacheInitializer.Succeeded, "Initialization of the second cache should have failed."); }