/// <inheritdoc />
public CreateSessionResult <ICacheSession> CreateSession(Context context, string name, ImplicitPin implicitPin)
{
return(Tracing.CreateSessionCall.Run(_tracer, context, name, () =>
{
var createContentResult = ContentStore.CreateSession(context, name, implicitPin);
if (!createContentResult.Succeeded)
{
return new CreateSessionResult <ICacheSession>(createContentResult, "Content session creation failed");
}
var contentSession = createContentResult.Session;
var createMemoizationResult = _passContentToMemoization
? MemoizationStore.CreateSession(context, name, contentSession)
: MemoizationStore.CreateSession(context, name);
if (!createMemoizationResult.Succeeded)
{
return new CreateSessionResult <ICacheSession>(createContentResult, "Memoization session creation failed");
}
var memoizationSession = createMemoizationResult.Session;
var session = new OneLevelCacheSession(name, implicitPin, memoizationSession, contentSession);
return new CreateSessionResult <ICacheSession>(session);
}));
}
/// <inheritdoc />
public virtual CreateSessionResult <IContentSession> CreateSession(Context context, string name, ImplicitPin implicitPin)
{
return(CreateSessionCall.Run(_tracer, OperationContext(context), name, () =>
{
var session = new FileSystemContentSession(name, implicitPin, Store);
return new CreateSessionResult <IContentSession>(session);
}));
}
/// <inheritdoc /> public abstract CreateSessionResult <IContentSession> CreateSession(Context context, string name, ImplicitPin implicitPin);
public CreateSessionResult <IContentSession> CreateSession(Context context, string name, ImplicitPin implicitPin)
{
return(new CreateSessionResult <IContentSession>(_contentSession));
}
/// <nodoc /> protected abstract CreateSessionResult <TSession> CreateSession(TStore store, OperationContext context, string name, ImplicitPin implicitPin);
private CreateSessionResult <T> CreateSession <T>(Context context, string name, ImplicitPin implicitPin)
where T : class, IName
{
var operationContext = OperationContext(context);
return(operationContext.PerformOperation(
Tracer,
() =>
{
var innerSessionResult = InnerStore.CreateSession(context, name, implicitPin).ThrowIfFailure();
var session = new VirtualizedContentSession(this, innerSessionResult.Session, name);
return new CreateSessionResult <T>(session as T);
}));
}
/// <inheritdoc /> public CreateSessionResult <ICacheSession> CreateSession(Context context, string name, ImplicitPin implicitPin) => _client.CreateSession(context, name, implicitPin);
public CreateSessionResult <ICacheSession> CreateSession(Context context, string name, ImplicitPin implicitPin)
{
return(Tracing.CreateSessionCall.Run(_tracer, context, name, () =>
{
var backingContentSessionResult = _backingContentStore.CreateSession(context, name, implicitPin);
if (!backingContentSessionResult.Succeeded)
{
return new CreateSessionResult <ICacheSession>(backingContentSessionResult);
}
IContentSession writeThroughContentSession = null;
if (_writeThroughContentStore != null)
{
var writeThroughContentSessionResult = _writeThroughContentStore.CreateSession(context, name, implicitPin);
if (!writeThroughContentSessionResult.Succeeded)
{
return new CreateSessionResult <ICacheSession>(writeThroughContentSessionResult);
}
writeThroughContentSession = writeThroughContentSessionResult.Session;
}
return new CreateSessionResult <ICacheSession>(
new BuildCacheSession(
_fileSystem,
name,
implicitPin,
_cacheNamespace,
Id,
_contentHashListAdapterFactory.Create(backingContentSessionResult.Session),
backingContentSessionResult.Session,
_maxFingerprintSelectorsToFetch,
_minimumTimeToKeepContentHashLists,
_rangeOfTimeToKeepContentHashLists,
_fingerprintIncorporationEnabled,
_maxDegreeOfParallelismForIncorporateRequests,
_maxFingerprintsPerIncorporateRequest,
writeThroughContentSession,
_sealUnbackedContentHashLists,
_overrideUnixFileAccessMode,
_tracer));
}));
}
public override CreateSessionResult <IContentSession> CreateSession(Context context, string name, ImplicitPin implicitPin)
{
return(CreateSessionCall.Run(ExecutionTracer, OperationContext(context), name, () =>
{
var session = new TestServiceClientContentSession(
name,
implicitPin,
Configuration.RetryPolicy,
_configuration.DataRootPath,
_overrideCacheName ?? Configuration.CacheName,
context.Logger,
FileSystem,
Configuration.Scenario,
this,
SessionTracer,
GetRpcConfig());
return new CreateSessionResult <IContentSession>(session);
}));
}
public CreateSessionResult <ICacheSession> CreateSession(Context context, string name, ImplicitPin implicitPin)
{
var innerSession = _innerICache.CreateSession(context, name, implicitPin);
var session = new DistributedCacheSession(_logger, name, innerSession.Session, _innerICache.Id, _metadataCache, _tracer, _readThroughMode);
return(new CreateSessionResult <ICacheSession>(session));
}
/// <nodoc />
public new virtual CreateSessionResult <ICacheSession> CreateSession(Context context, string name, ImplicitPin implicitPin)
{
var operationContext = OperationContext(context);
return(operationContext.PerformOperation(
Tracer,
() =>
{
var session = new ServiceClientCacheSession(name, implicitPin, Logger, FileSystem, SessionTracer, Configuration);
return new CreateSessionResult <ICacheSession>(session);
}));
}
/// <inheritdoc /> protected override CreateSessionResult <IContentSession> CreateSession(IContentStore store, OperationContext context, string name, ImplicitPin implicitPin) => store.CreateSession(context, name, implicitPin);
internal void IncorporateStrongFingerprints(
[Required, Description("Path to log containing the strong fingerpints")] string log,
[Required, Description("BuildCache url to which to incorporate the fingerprints")] string buildCacheUrl,
[Required, Description("BlobStore url associated to the buildCache url")] string blobStoreUrl,
[Required, Description("Cache namespace to target within BuildCache")] string cacheNamespace,
[Required, Description("Whether or not to use the newer Blob implementation of BuildCache")] bool useBlobContentHashLists,
[DefaultValue(false), Description("Whether or not to use aad authentication")] bool useAad,
[DefaultValue(null), Description("Maximum number of fingerprint batches sent in parallel within a set. Default: System.Environment.ProcessorCount")] int?maxDegreeOfParallelism,
[DefaultValue(null), Description("Maximum number of fingerprints per batch within a set. Default: 500")] int?maxFingerprintsPerRequest,
[DefaultValue(1), Description("How many times to incorporate the set of fingerprints")] int iterationCount,
[DefaultValue(1), Description("How many sets of fingerprints to incorporate in parallel")] int iterationDegreeOfParallelism)
{
Initialize();
Stopwatch stopwatch = Stopwatch.StartNew();
int count = 0;
AbsolutePath logPath = new AbsolutePath(log);
if (!_fileSystem.FileExists(logPath))
{
throw new ArgumentException($"Log file {log} does not exist.", nameof(log));
}
BuildCacheServiceConfiguration config = new BuildCacheServiceConfiguration(buildCacheUrl, blobStoreUrl)
{
CacheNamespace = cacheNamespace,
UseAad = useAad,
UseBlobContentHashLists = useBlobContentHashLists,
FingerprintIncorporationEnabled = true
};
if (maxDegreeOfParallelism.HasValue)
{
config.MaxDegreeOfParallelismForIncorporateRequests = maxDegreeOfParallelism.Value;
}
if (maxFingerprintsPerRequest.HasValue)
{
config.MaxFingerprintsPerIncorporateRequest = maxFingerprintsPerRequest.Value;
}
using (var logStream = _fileSystem.OpenReadOnlySafeAsync(logPath, FileShare.Read | FileShare.Delete).Result)
using (StreamReader reader = new StreamReader(logStream))
{
Context context = new Context(_logger);
RunBuildCacheAsync(
context,
config,
async cache =>
{
const string sessionName = "CommandLineSessionName";
const ImplicitPin implicitPin = ImplicitPin.None;
List <StrongFingerprint> strongFingerprints = EnumerateUniqueStrongFingerprints(reader).ToList();
count = strongFingerprints.Count;
_logger.Always($"Incorporating {count} strong fingerprints {iterationCount} times");
ActionBlock <List <StrongFingerprint> > incorporateBlock =
new ActionBlock <List <StrongFingerprint> >(
async fingerprints =>
{
var iterationStopwatch = Stopwatch.StartNew();
var iterationContext = context.CreateNested();
CreateSessionResult <ICacheSession> createSessionResult = cache.CreateSession(
iterationContext, sessionName, implicitPin);
if (!createSessionResult.Succeeded)
{
iterationContext.TraceMessage(Severity.Error, $"Failed to create BuildCache session. Result=[{createSessionResult}]");
return;
}
using (ICacheSession session = createSessionResult.Session)
{
BoolResult r = await session.StartupAsync(iterationContext);
if (!r.Succeeded)
{
iterationContext.TraceMessage(Severity.Error, $"Failed to start up BuildCache client session. Result=[{r}]");
return;
}
try
{
await session.IncorporateStrongFingerprintsAsync(
iterationContext, strongFingerprints.AsTasks(), CancellationToken.None).IgnoreFailure();
}
finally
{
iterationStopwatch.Stop();
r = await session.ShutdownAsync(iterationContext);
if (r.Succeeded)
{
_logger.Always(
$"Incorporated {count} fingerprints in {iterationStopwatch.ElapsedMilliseconds / 1000} seconds.");
}
else
{
iterationContext.TraceMessage(Severity.Error, $"Failed to shut down BuildCache client Session. Result=[{r}]");
}
}
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = iterationDegreeOfParallelism
});
for (int i = 0; i < iterationCount; i++)
{
await incorporateBlock.SendAsync(strongFingerprints);
}
incorporateBlock.Complete();
await incorporateBlock.Completion;
}).Wait();
}
stopwatch.Stop();
_logger.Always($"Incorporated {count} unique strong fingerprints {iterationCount} times in {stopwatch.ElapsedMilliseconds / 1000} seconds.");
}
protected async Task RunTestAsync(
Context context,
int storeCount,
Func <TestContext, Task> testFunc,
ImplicitPin implicitPin = ImplicitPin.PutAndGet,
bool enableDistributedEviction = false,
int?replicaCreditInMinutes = null,
bool enableRepairHandling = false,
int iterations = 1)
{
var additionalArgs = PrepareAdditionalCreateStoreArgs(storeCount);
var indexedDirectories = Enumerable.Range(0, storeCount)
.Select(i => new { Index = i, Directory = new DisposableDirectory(FileSystem, TestRootDirectoryPath / i.ToString()) })
.ToList();
var testFileCopier = new TestFileCopier()
{
WorkingDirectory = indexedDirectories[0].Directory.Path
};
for (int iteration = 0; iteration < iterations; iteration++)
{
var stores = indexedDirectories.Select(
directory =>
CreateStore(
context,
testFileCopier,
directory.Directory,
directory.Index,
enableDistributedEviction,
replicaCreditInMinutes,
enableRepairHandling,
additionalArgs)).ToList();
var startupResults = await TaskSafetyHelpers.WhenAll(stores.Select(async store => await store.StartupAsync(context)));
Assert.True(startupResults.All(x => x.Succeeded), $"Failed to startup: {string.Join(Environment.NewLine, startupResults.Where(s => !s))}");
var id = 0;
var sessions = stores.Select(store => store.CreateSession(context, "store" + id++, implicitPin).Session).ToList();
await TaskSafetyHelpers.WhenAll(sessions.Select(async session => await session.StartupAsync(context)));
var testContext = new TestContext(context, testFileCopier, indexedDirectories.Select(p => p.Directory).ToList(), sessions, stores, iteration);
await testFunc(testContext);
await TaskSafetyHelpers.WhenAll(
sessions.Select(async session =>
{
if (!session.ShutdownCompleted)
{
await session.ShutdownAsync(context).ThrowIfFailure();
}
}));
sessions.ForEach(session => session.Dispose());
await TaskSafetyHelpers.WhenAll(Enumerable.Range(0, storeCount).Select(storeId => LogStats(testContext, storeId)));
await TaskSafetyHelpers.WhenAll(stores.Select(async store => await store.ShutdownAsync(context)));
stores.ForEach(store => store.Dispose());
}
indexedDirectories.ForEach(directory => directory.Directory.Dispose());
}
/// <inheritdoc />
public CreateSessionResult <IContentSession> CreateSession(Context context, string name, ImplicitPin implicitPin)
{
return(CreateSessionCall.Run(_tracer, OperationContext(context), name, () =>
{
CreateSessionResult <IContentSession> innerSessionResult = InnerContentStore.CreateSession(context, name, implicitPin);
if (innerSessionResult.Succeeded)
{
var session = new DistributedContentSession <T>(
name,
innerSessionResult.Session,
_contentLocationStore,
_distributedCopier,
LocalMachineLocation,
pinCache: _pinCache,
contentTrackerUpdater: _contentTrackerUpdater,
settings: _settings);
return new CreateSessionResult <IContentSession>(session);
}
return new CreateSessionResult <IContentSession>(innerSessionResult, "Could not initialize inner content session with error");
}));
}
/// <nodoc />
public CreateSessionResult <IContentSession> CreateSession(Context context, string name, ImplicitPin implicitPin)
{
return(CreateSessionCall.Run(Tracer, new OperationContext(context), name, () =>
{
var sessions = new Dictionary <string, IReadOnlyContentSession>(StringComparer.OrdinalIgnoreCase);
foreach (KeyValuePair <string, IContentStore> entry in _drivesWithContentStore)
{
var result = entry.Value.CreateSession(context, name, implicitPin);
if (!result.Succeeded)
{
foreach (var session in sessions.Values)
{
session.Dispose();
}
return new CreateSessionResult <IContentSession>(result);
}
sessions.Add(entry.Key, result.Session);
}
var multiCacheSession = new MultiplexedContentSession(SessionTracer, sessions, name, _preferredCacheDrive);
return new CreateSessionResult <IContentSession>(multiCacheSession);
}));
}
/// <inheritdoc />
public override CreateSessionResult <IContentSession> CreateSession(Context context, string name, ImplicitPin implicitPin)
{
return(CreateSessionCall.Run(_tracer, new OperationContext(context), name, () =>
{
var sessionForStream = ContentStoreForStream.CreateSession(context, name, implicitPin);
if (!sessionForStream.Succeeded)
{
return new CreateSessionResult <IContentSession>(sessionForStream, "creation of stream content session failed");
}
var sessionForPath = ContentStoreForPath.CreateSession(context, name, implicitPin);
if (!sessionForPath.Succeeded)
{
return new CreateSessionResult <IContentSession>(sessionForPath, "creation of path content session failed");
}
var session = new StreamPathContentSession(
name,
sessionForStream.Session,
sessionForPath.Session);
return new CreateSessionResult <IContentSession>(session);
}));
}
/// <inheritdoc />
public CreateSessionResult <IReadOnlyContentSession> CreateReadOnlySession(Context context, string name, ImplicitPin implicitPin)
{
return(CreateSession <IReadOnlyContentSession>(context, name, implicitPin));
}
