/// <summary>
/// Returns a cache initializer to a real instance of the cache
/// </summary>
protected CacheInitializer GetRealCacheInitializerForTests()
{
var tempDir = OperatingSystemHelper.IsUnixOS ? "/tmp/buildxl-temp" : TemporaryDirectory;
string cacheDirectory = Path.Combine(tempDir, "cache");
AbsolutePath cacheConfigPath = WriteTestCacheConfigToDisk(cacheDirectory);
var translator = new RootTranslator();
if (TryGetSubstSourceAndTarget(out var substSource, out var substTarget))
{
translator.AddTranslation(substTarget, substSource);
}
translator.Seal();
Configuration.Cache.CacheConfigFile = cacheConfigPath;
Configuration.Cache.CacheLogFilePath = AbsolutePath.Create(Context.PathTable, tempDir).Combine(Context.PathTable, "cache.log");
var maybeCacheInitializer = CacheInitializer.GetCacheInitializationTask(
LoggingContext,
Context.PathTable,
cacheDirectory,
Configuration.Cache,
translator,
recoveryStatus: false,
cancellationToken: CancellationToken.None).GetAwaiter().GetResult();
if (!maybeCacheInitializer.Succeeded)
{
throw new BuildXLException("Unable to initialize the real cache: " + maybeCacheInitializer.Failure.DescribeIncludingInnerFailures());
}
return(maybeCacheInitializer.Result);
}
public void RootTranslation()
{
var rootTranslator = new RootTranslator();
var pathTable = Context.PathTable;
// Source is shorter than target
var shortCaseSourceTranslatedRootPath = GetFullPath(pathTable, "ShortSource").Expand(pathTable);
var shortCaseTargetTranslatedRootPath = GetFullPath(pathTable, "Short___Target").Expand(pathTable);
rootTranslator.AddTranslation(
shortCaseSourceTranslatedRootPath.ExpandedPath,
shortCaseTargetTranslatedRootPath.ExpandedPath);
// Source is longer than target
var longCaseSourceTranslatedRootPath = GetFullPath(pathTable, "LongSourceSource").Expand(pathTable);
var longCaseTargetTranslatedRootPath = GetFullPath(pathTable, "LongTarget").Expand(pathTable);
rootTranslator.AddTranslation(
longCaseSourceTranslatedRootPath.ExpandedPath,
longCaseTargetTranslatedRootPath.ExpandedPath);
rootTranslator.Seal();
var cache = new CacheCoreArtifactContentCache(Session, rootTranslator);
// SHORTER SOURCE, SAME ROOT: Path should NOT be translated
VerifyExpandedPathForCacheEquals(cache, shortCaseSourceTranslatedRootPath, shortCaseSourceTranslatedRootPath);
// LONGER SOURCE, SAME ROOT: Path SHOULD be translated
VerifyExpandedPathForCacheEquals(cache, longCaseSourceTranslatedRootPath, longCaseTargetTranslatedRootPath);
}
/// <nodoc />
public CacheCoreArtifactContentCache(
ICacheSession cache,
RootTranslator rootTranslator)
{
m_cache = new PossiblyOpenCacheSession(cache);
m_rootTranslator = rootTranslator;
}
/// <nodoc />
public CacheCoreArtifactContentCache(
ICacheSession cache,
RootTranslator rootTranslator,
bool replaceExistingFileOnMaterialization = false)
{
m_cache = new PossiblyOpenCacheSession(cache);
m_rootTranslator = rootTranslator;
m_replaceExistingFileOnMaterialization = replaceExistingFileOnMaterialization;
}
/// <summary>
/// Gets an instance of <see cref="ICacheConfigData"/> from cache configuration.
/// </summary>
internal static Possible <ICacheConfigData> TryGetCacheConfigData(
PathTable pathTable,
string cacheDirectory,
ICacheConfiguration config,
RootTranslator rootTranslator = null)
{
Contract.Requires(pathTable != null);
Contract.Requires(pathTable.IsValid);
Contract.Requires(config != null);
Contract.Requires(config.CacheLogFilePath.IsValid);
Contract.Requires(config.CacheConfigFile.IsValid);
Contract.Requires(!string.IsNullOrWhiteSpace(cacheDirectory));
Possible <string> maybeConfigData = TryReadCacheConfigFile(config.CacheConfigFile.ToString(pathTable));
if (!maybeConfigData.Succeeded)
{
return(maybeConfigData.Failure);
}
// Update the cache config to dynamically set the cache path if it is configured to use the per-invocation path.
// TODO: Ideally this would be exposed as config constructor parameters to BuildXL to not require manipulating the json config.
// But for now we just modify the config text before passing it along to the cache.
string cacheConfigContent = maybeConfigData.Result;
cacheConfigContent = cacheConfigContent.Replace("[DominoSelectedLogPath]", config.CacheLogFilePath.ToString(pathTable).Replace(@"\", @"\\")); // Escape path separation chars to json format
cacheConfigContent = cacheConfigContent.Replace("[BuildXLSelectedLogPath]", config.CacheLogFilePath.ToString(pathTable).Replace(@"\", @"\\")); // Escape path separation chars to json format
cacheConfigContent = cacheConfigContent.Replace("[DominoSelectedRootPath]", cacheDirectory.Replace(@"\", @"\\"));
cacheConfigContent = cacheConfigContent.Replace("[BuildXLSelectedRootPath]", cacheDirectory.Replace(@"\", @"\\"));
cacheConfigContent = cacheConfigContent.Replace("[UseDedupStore]", config.UseDedupStore.ToString());
cacheConfigContent = cacheConfigContent.Replace("[ReplaceExistingFileOnMaterialization]", config.ReplaceExistingFileOnMaterialization.ToString());
var vfsCasRoot = config.VfsCasRoot.IsValid
? config.VfsCasRoot.ToString(pathTable)
: "";
if (rootTranslator != null && !string.IsNullOrEmpty(vfsCasRoot))
{
// VFS needs real path so use root translator to resolve to real path.
vfsCasRoot = rootTranslator.Translate(vfsCasRoot);
}
// Escape path separation chars to json format
vfsCasRoot = vfsCasRoot.Replace(@"\", @"\\");
cacheConfigContent = cacheConfigContent.Replace("[VfsCasRoot]", vfsCasRoot);
ICacheConfigData cacheConfigData;
Exception exception;
if (!CacheFactory.TryCreateCacheConfigData(cacheConfigContent, out cacheConfigData, out exception))
{
return(new Failure <string>(I($"Unable to create cache config data: {exception.GetLogEventMessage()}")));
}
return(new Possible <ICacheConfigData>(cacheConfigData));
}
private CacheCoreCacheInitializer(
LoggingContext loggingContext,
ICacheCoreCache cache,
ICacheCoreSession session,
List <IDisposable> acquiredDisposables,
bool enableFingerprintLookup,
RootTranslator rootTranslator)
: base(
loggingContext,
acquiredDisposables,
enableFingerprintLookup)
{
Contract.Requires(cache != null);
Contract.Requires(session != null);
m_cache = cache;
m_session = session;
m_rootTranslator = rootTranslator;
m_initialStatistics = GetCacheBulkStatistics(session);
}
public void RootTranslationDifferentRoots()
{
var rootTranslator = new RootTranslator();
var pathTable = Context.PathTable;
// Source is shorter than target but has different root
var shortCaseChangeRootSourceTranslatedRootPath = ChangeRoot(GetFullPath(pathTable, "ShortChangeRootSrc"), pathTable, newRoot: 'A');
var shortCaseChangeRootTargetTranslatedRootPath = ChangeRoot(GetFullPath(pathTable, "ShortChangeRootTarget"), pathTable, newRoot: 'B');
rootTranslator.AddTranslation(
shortCaseChangeRootSourceTranslatedRootPath.ExpandedPath,
shortCaseChangeRootTargetTranslatedRootPath.ExpandedPath);
rootTranslator.Seal();
var cache = new CacheCoreArtifactContentCache(Session, rootTranslator);
// SHORTER SOURCE, DIFFERENT ROOT: Path SHOULD be translated
VerifyExpandedPathForCacheEquals(cache, shortCaseChangeRootSourceTranslatedRootPath, shortCaseChangeRootTargetTranslatedRootPath);
}
public static CacheInitializationTask GetCacheInitializationTask(
LoggingContext loggingContext,
PathTable pathTable,
string cacheDirectory,
ICacheConfiguration config,
RootTranslator rootTranslator,
bool?recoveryStatus,
CancellationToken cancellationToken,
// Only used for testing purposes to inject cache.
Func <EngineCache> testHookCacheFactory = null)
{
Contract.Requires(recoveryStatus.HasValue, "Recovery attempt should have been done before initializing the cache");
DateTime startTime = DateTime.UtcNow;
var task = Task.Run(
async() =>
{
using (PerformanceMeasurement.Start(
loggingContext,
"CacheInitialization",
Tracing.Logger.Log.StartInitializingCache,
Tracing.Logger.Log.EndInitializingCache))
{
if (testHookCacheFactory != null)
{
return(new MemoryCacheInitializer(
testHookCacheFactory,
loggingContext,
new List <IDisposable>(),
enableFingerprintLookup: config.Incremental));
}
Possible <CacheCoreCacheInitializer> maybeCacheCoreEngineCache =
await CacheCoreCacheInitializer.TryInitializeCacheInternalAsync(
loggingContext,
pathTable,
cacheDirectory,
config,
enableFingerprintLookup: config.Incremental,
rootTranslator: rootTranslator);
if (!maybeCacheCoreEngineCache.Succeeded)
{
string errorMessage = maybeCacheCoreEngineCache.Failure.Describe();
if (errorMessage.Contains(LockAcquisitionFailureMessagePrefix))
{
Tracing.Logger.Log.FailedToAcquireDirectoryLock(
loggingContext,
maybeCacheCoreEngineCache.Failure.DescribeIncludingInnerFailures());
}
else
{
Tracing.Logger.Log.StorageCacheStartupError(
loggingContext,
maybeCacheCoreEngineCache.Failure.DescribeIncludingInnerFailures());
}
}
return(maybeCacheCoreEngineCache.Then <CacheInitializer>(c => c));
}
}, cancellationToken);
return(new CacheInitializationTask(
loggingContext,
startTime,
task,
cancellationToken));
}
internal static async Task <Possible <CacheCoreCacheInitializer> > TryInitializeCacheInternalAsync(
LoggingContext loggingContext,
PathTable pathTable,
string cacheDirectory,
ICacheConfiguration config,
bool enableFingerprintLookup,
RootTranslator rootTranslator)
{
Contract.Requires(pathTable != null);
Contract.Requires(pathTable.IsValid);
Contract.Requires(config != null);
Contract.Requires(config.CacheLogFilePath.IsValid);
Contract.Requires(config.CacheConfigFile.IsValid);
Contract.Requires(!string.IsNullOrWhiteSpace(cacheDirectory));
bool succeeded = false;
ICacheCoreCache cache = null;
ICacheCoreSession session = null;
try
{
Possible <ICacheConfigData> cacheConfigData = TryGetCacheConfigData(pathTable, cacheDirectory, config, rootTranslator);
if (!cacheConfigData.Succeeded)
{
return(cacheConfigData.Failure);
}
Possible <ICacheCoreCache> maybeCache = await CacheFactory.InitializeCacheAsync(cacheConfigData.Result, loggingContext.ActivityId, config);
if (!maybeCache.Succeeded)
{
return(maybeCache.Failure);
}
// We are now responsible for shutting this down (even if something later fails).
cache = maybeCache.Result;
cache.SuscribeForCacheStateDegredationFailures(
failure => { Tracing.Logger.Log.CacheReportedRecoverableError(loggingContext, failure.DescribeIncludingInnerFailures()); });
// Log the cache ID we got.
Tracing.Logger.Log.CacheInitialized(loggingContext, cache.CacheId);
Possible <ICacheCoreSession> maybeSession =
string.IsNullOrWhiteSpace(config.CacheSessionName)
? await cache.CreateSessionAsync()
: await cache.CreateSessionAsync(config.CacheSessionName);
if (!maybeSession.Succeeded)
{
return(maybeSession.Failure);
}
session = maybeSession.Result;
succeeded = true;
return(new CacheCoreCacheInitializer(
loggingContext,
cache,
session,
new List <IDisposable>(),
enableFingerprintLookup: enableFingerprintLookup,
rootTranslator: rootTranslator,
replaceExistingFileOnMaterialization: config.ReplaceExistingFileOnMaterialization));
}
finally
{
if (!succeeded)
{
// Note that we clean up in reverse order that we initialized things.
if (session != null)
{
Analysis.IgnoreResult(await session.CloseAsync(), justification: "Okay to ignore close");
Analysis.IgnoreResult(await cache.ShutdownAsync(), justification: "Okay to ignore shutdown");
}
}
}
}
public void TestProperLogMessageOnCacheLockAcquisitionFailure()
{
var tempDir = Path.Combine(
OperatingSystemHelper.IsUnixOS ? "/tmp/bxl-temp" : TemporaryDirectory,
Guid.NewGuid().ToString());
string cacheDirectory = Path.Combine(tempDir, "cache");
string cacheConfigJson = $@"{{
""MaxCacheSizeInMB"": 1024,
""CacheId"": ""TestCache"",
""Assembly"": ""BuildXL.Cache.MemoizationStoreAdapter"",
""CacheLogPath"": ""[BuildXLSelectedLogPath]"",
""Type"": ""BuildXL.Cache.MemoizationStoreAdapter.MemoizationStoreCacheFactory"",
""CacheRootPath"": ""{cacheDirectory.Replace("\\", "\\\\")}"",
""UseStreamCAS"": false,
""SingleInstanceTimeoutInSeconds"" : 5
}}";
AbsolutePath cacheConfigPath = WriteTestCacheConfigToDisk(cacheDirectory, cacheConfigJson);
var translator = new RootTranslator();
translator.Seal();
var possibleFirstCacheInitializer = CacheInitializer.GetCacheInitializationTask(
LoggingContext,
Context.PathTable,
cacheDirectory,
new CacheConfiguration
{
CacheLogFilePath = AbsolutePath.Create(Context.PathTable, tempDir).Combine(Context.PathTable, "cache.log"),
CacheConfigFile = cacheConfigPath
},
translator,
recoveryStatus: false,
cancellationToken: CancellationToken.None).GetAwaiter().GetResult();
if (!possibleFirstCacheInitializer.Succeeded)
{
AssertTrue(false, "Failed to initialize the cache: " + possibleFirstCacheInitializer.Failure.DescribeIncludingInnerFailures());
}
var possibleSecondCacheInitializer = CacheInitializer.GetCacheInitializationTask(
LoggingContext,
Context.PathTable,
cacheDirectory,
new CacheConfiguration
{
// need a different name for the log file (due to the order in which the things are initialized)
CacheLogFilePath = AbsolutePath.Create(Context.PathTable, tempDir).Combine(Context.PathTable, "cache_2.log"),
CacheConfigFile = cacheConfigPath,
},
translator,
recoveryStatus: false,
cancellationToken: CancellationToken.None).GetAwaiter().GetResult();
// close and dispose the first cache (must be done before the assert block bellow)
var firstCacheInitializer = possibleFirstCacheInitializer.Result;
AssertSuccess(firstCacheInitializer.Close());
firstCacheInitializer.Dispose();
AssertErrorEventLogged(global::BuildXL.Engine.Tracing.LogEventId.FailedToAcquireDirectoryLock);
AssertTrue(!possibleSecondCacheInitializer.Succeeded, "Initialization of the second cache should have failed.");
}
