private async Task RunBuildCacheAsync(
Context context, BuildCacheServiceConfiguration config, Func <ICache, Task> funcAsync)
{
VssCredentialsFactory credentialsFactory = new VssCredentialsFactory(new VsoCredentialHelper());
ICache cache = BuildCacheCacheFactory.Create(_fileSystem, _logger, credentialsFactory, config, null);
using (cache)
{
BoolResult r = await cache.StartupAsync(context);
if (!r.Succeeded)
{
_tracer.Error(context, $"Failed to start up BuildCache client. Result=[{r}]");
return;
}
try
{
await funcAsync(cache);
}
finally
{
r = await cache.ShutdownAsync(context);
if (!r.Succeeded)
{
_tracer.Error(context, $"Failed to shut down BuildCache client. Result=[{r}]");
}
}
}
}
internal static BuildXL.Cache.MemoizationStore.Interfaces.Caches.ICache CreateBuildCacheCache <T>(T cacheConfig, ILogger logger, string pat = null) where T : BuildCacheCacheConfig
{
// TODO: Remove check when all clients are updated with unified Dedup flag
if ((ContentHashingUtilities.HashInfo.HashType.IsValidDedup()) ^ cacheConfig.UseDedupStore)
{
var store = cacheConfig.UseDedupStore ? "DedupStore" : "BlobStore";
throw new ArgumentException($"HashType {ContentHashingUtilities.HashInfo.HashType} cannot be used with {store}");
}
var credentialProviderHelper = new CredentialProviderHelper(m => logger.Debug(m));
if (credentialProviderHelper.IsCredentialProviderSpecified())
{
logger.Debug($"Credential providers path specified: '{credentialProviderHelper.CredentialHelperPath}'");
}
else
{
logger.Debug("Using current user's credentials for obtaining AAD token");
}
var credentialsFactory = new VssCredentialsFactory(pat, credentialProviderHelper, m => logger.Debug(m));
logger.Diagnostic("Creating BuildCacheCache factory");
var fileSystem = new PassThroughFileSystem(logger);
// TODO: Once write-behind is implemented send a contentstorefunc down to the create.
Func <IContentStore> writeThroughStore = null;
if (!string.IsNullOrWhiteSpace(cacheConfig.CacheName))
{
ServiceClientRpcConfiguration rpcConfiguration;
if (cacheConfig.GrpcPort != 0)
{
rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort);
}
else
{
var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName);
var portReader = factory.GetPortReader();
var port = portReader.ReadPort();
rpcConfiguration = new ServiceClientRpcConfiguration(port);
}
writeThroughStore = () =>
new ServiceClientContentStore(
logger,
fileSystem,
cacheConfig.CacheName,
rpcConfiguration,
cacheConfig.ConnectionRetryIntervalSeconds,
cacheConfig.ConnectionRetryCount,
scenario: cacheConfig.ScenarioName);
}
BuildCacheServiceConfiguration buildCacheServiceConfiguration = cacheConfig.AsBuildCacheServiceConfigurationFile();
return(BuildCacheCacheFactory.Create(fileSystem, logger, credentialsFactory, buildCacheServiceConfiguration, writeThroughStore));
}
protected override ICachePublisher CreatePublisher(
string sessionName,
BuildCacheServiceConfiguration config,
string pat,
Context context)
{
sessionName.Should().NotBeNull();
return(new DummyPublisher(config.ForceUpdateOnAddContentHashList));
}
public BuildCacheTestPublishingSession(
BuildCacheServiceConfiguration config,
string name,
string pat,
IContentSession contentSource,
IAbsFileSystem fileSystem,
SemaphoreSlim publishingGate)
: base(config, name, pat, contentSource, fileSystem, publishingGate)
{
}
internal void IncorporateStrongFingerprints(
[Required, Description("Path to log containing the strong fingerpints")] string log,
[Required, Description("BuildCache url to which to incorporate the fingerprints")] string buildCacheUrl,
[Required, Description("BlobStore url associated to the buildCache url")] string blobStoreUrl,
[Required, Description("Cache namespace to target within BuildCache")] string cacheNamespace,
[Required, Description("Whether or not to use the newer Blob implementation of BuildCache")] bool useBlobContentHashLists,
[DefaultValue(false), Description("Whether or not to use aad authentication")] bool useAad,
[DefaultValue(null), Description("Maximum number of fingerprint batches sent in parallel within a set. Default: System.Environment.ProcessorCount")] int?maxDegreeOfParallelism,
[DefaultValue(null), Description("Maximum number of fingerprints per batch within a set. Default: 500")] int?maxFingerprintsPerRequest,
[DefaultValue(1), Description("How many times to incorporate the set of fingerprints")] int iterationCount,
[DefaultValue(1), Description("How many sets of fingerprints to incorporate in parallel")] int iterationDegreeOfParallelism)
{
Initialize();
Stopwatch stopwatch = Stopwatch.StartNew();
int count = 0;
AbsolutePath logPath = new AbsolutePath(log);
if (!_fileSystem.FileExists(logPath))
{
throw new ArgumentException($"Log file {log} does not exist.", nameof(log));
}
BuildCacheServiceConfiguration config = new BuildCacheServiceConfiguration(buildCacheUrl, blobStoreUrl)
{
CacheNamespace = cacheNamespace,
UseAad = useAad,
UseBlobContentHashLists = useBlobContentHashLists,
FingerprintIncorporationEnabled = true
};
if (maxDegreeOfParallelism.HasValue)
{
config.MaxDegreeOfParallelismForIncorporateRequests = maxDegreeOfParallelism.Value;
}
if (maxFingerprintsPerRequest.HasValue)
{
config.MaxFingerprintsPerIncorporateRequest = maxFingerprintsPerRequest.Value;
}
using (var logStream = _fileSystem.OpenReadOnlySafeAsync(logPath, FileShare.Read | FileShare.Delete).Result)
using (StreamReader reader = new StreamReader(logStream))
{
Context context = new Context(_logger);
RunBuildCacheAsync(
context,
config,
async cache =>
{
const string sessionName = "CommandLineSessionName";
const ImplicitPin implicitPin = ImplicitPin.None;
List <StrongFingerprint> strongFingerprints = EnumerateUniqueStrongFingerprints(reader).ToList();
count = strongFingerprints.Count;
_logger.Always($"Incorporating {count} strong fingerprints {iterationCount} times");
ActionBlock <List <StrongFingerprint> > incorporateBlock =
new ActionBlock <List <StrongFingerprint> >(
async fingerprints =>
{
var iterationStopwatch = Stopwatch.StartNew();
var iterationContext = context.CreateNested(nameof(IncorporateStrongFingerprints));
CreateSessionResult <ICacheSession> createSessionResult = cache.CreateSession(
iterationContext, sessionName, implicitPin);
if (!createSessionResult.Succeeded)
{
_tracer.Error(iterationContext, $"Failed to create BuildCache session. Result=[{createSessionResult}]");
return;
}
using (ICacheSession session = createSessionResult.Session)
{
BoolResult r = await session.StartupAsync(iterationContext);
if (!r.Succeeded)
{
_tracer.Error(iterationContext, $"Failed to start up BuildCache client session. Result=[{r}]");
return;
}
try
{
await session.IncorporateStrongFingerprintsAsync(
iterationContext, strongFingerprints.AsTasks(), CancellationToken.None).IgnoreFailure();
}
finally
{
iterationStopwatch.Stop();
r = await session.ShutdownAsync(iterationContext);
if (r.Succeeded)
{
_tracer.Always(context,
$"Incorporated {count} fingerprints in {iterationStopwatch.ElapsedMilliseconds / 1000} seconds.");
}
else
{
_tracer.Error(iterationContext, $"Failed to shut down BuildCache client Session. Result=[{r}]");
}
}
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = iterationDegreeOfParallelism
});
for (int i = 0; i < iterationCount; i++)
{
await incorporateBlock.SendAsync(strongFingerprints);
}
incorporateBlock.Complete();
await incorporateBlock.Completion;
}).Wait();
}
stopwatch.Stop();
_logger.Always($"Incorporated {count} unique strong fingerprints {iterationCount} times in {stopwatch.ElapsedMilliseconds / 1000} seconds.");
}
internal static BuildXL.Cache.MemoizationStore.Interfaces.Caches.ICache CreateBuildCacheCache <T>(T cacheConfig, ILogger logger, string pat = null) where T : BuildCacheCacheConfig
{
// TODO: Remove check when all clients are updated with unified Dedup flag
if (ContentHashingUtilities.HashInfo.HashType == HashType.DedupNodeOrChunk ^ cacheConfig.UseDedupStore)
{
var store = cacheConfig.UseDedupStore ? "DedupStore" : "BlobStore";
throw new ArgumentException($"HashType {ContentHashingUtilities.HashInfo.HashType} cannot be used with {store}");
}
string credentialProviderPath = Environment.GetEnvironmentVariable(CredentialProvidersPathEnvVariable);
if (!string.IsNullOrWhiteSpace(credentialProviderPath))
{
logger.Debug($"Credential providers path specified: {credentialProviderPath}");
}
else
{
logger.Debug("Using current user's credentials for obtaining AAD token");
}
VssCredentialsFactory credentialsFactory;
#if !PLATFORM_OSX
string userName = null; // when running on .NET Framework, user name doesn't have to be explicitly provided
#if FEATURE_CORECLR
userName = GetAadUserNameUpn();
#endif
credentialsFactory = new VssCredentialsFactory(new VsoCredentialHelper(s => logger.Debug(s)), userName);
#else
var secPat = new SecureString();
if (!string.IsNullOrWhiteSpace(pat))
{
foreach (char c in pat)
{
secPat.AppendChar(c);
}
}
else
{
throw new ArgumentException("PAT must be supplied when running with CoreCLR");
}
credentialsFactory = new VssCredentialsFactory(new VssBasicCredential(new NetworkCredential(string.Empty, secPat)));
#endif
logger.Diagnostic("Creating BuildCacheCache factory");
var fileSystem = new PassThroughFileSystem(logger);
// TODO: Once write-behind is implemented send a contentstorefunc down to the create.
Func <IContentStore> writeThroughStore = null;
if (!string.IsNullOrWhiteSpace(cacheConfig.CacheName))
{
ServiceClientRpcConfiguration rpcConfiguration;
if (cacheConfig.GrpcPort != 0)
{
rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort);
}
else
{
var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName);
var portReader = factory.GetPortReader();
var port = portReader.ReadPort();
rpcConfiguration = new ServiceClientRpcConfiguration(port);
}
writeThroughStore = () =>
new ServiceClientContentStore(
logger,
fileSystem,
cacheConfig.CacheName,
rpcConfiguration,
cacheConfig.ConnectionRetryIntervalSeconds,
cacheConfig.ConnectionRetryCount,
scenario: cacheConfig.ScenarioName);
}
BuildCacheServiceConfiguration buildCacheServiceConfiguration = cacheConfig.AsBuildCacheServiceConfigurationFile();
return(BuildCacheCacheFactory.Create(fileSystem, logger, credentialsFactory, buildCacheServiceConfiguration, writeThroughStore));
}
internal static BuildXL.Cache.MemoizationStore.Interfaces.Caches.ICache CreateBuildCacheCache <T>(T cacheConfig, ILogger logger, string pat = null) where T : BuildCacheCacheConfig
{
// TODO: Remove check when all clients are updated with unified Dedup flag
if ((ContentHashingUtilities.HashInfo.HashType.IsValidDedup()) ^ cacheConfig.UseDedupStore)
{
var store = cacheConfig.UseDedupStore ? "DedupStore" : "BlobStore";
throw new ArgumentException($"HashType {ContentHashingUtilities.HashInfo.HashType} cannot be used with {store}");
}
string credentialProviderPath = Environment.GetEnvironmentVariable(CredentialProvidersPathEnvVariable);
bool isCredentialProviderSpecified = !string.IsNullOrWhiteSpace(credentialProviderPath);
if (isCredentialProviderSpecified)
{
logger.Debug($"Credential providers path specified: {credentialProviderPath}");
}
else
{
logger.Debug("Using current user's credentials for obtaining AAD token");
}
VssCredentialsFactory credentialsFactory;
#if PLATFORM_WIN
// Obtain and explicitly specify AAD user name ONLY when
// (1) no credential provider is specified, and
// (2) running on .NET Core.
// When a credential provider is specified, specifying AAD user name will override it and we don't want to do that.
// When running on .NET Framework, VsoCredentialHelper will automatically obtain currently logged on AAD user name.
string userName = !isCredentialProviderSpecified && Utilities.OperatingSystemHelper.IsDotNetCore
? GetAadUserNameUpn()
: null;
credentialsFactory = new VssCredentialsFactory(new VsoCredentialHelper(s => logger.Debug(s)), userName);
#else
var secPat = new SecureString();
if (!string.IsNullOrWhiteSpace(pat))
{
foreach (char c in pat)
{
secPat.AppendChar(c);
}
}
else
{
throw new ArgumentException("PAT must be supplied when not running on Windows");
}
credentialsFactory = new VssCredentialsFactory(new VssBasicCredential(new NetworkCredential(string.Empty, secPat)));
#endif
logger.Diagnostic("Creating BuildCacheCache factory");
var fileSystem = new PassThroughFileSystem(logger);
// TODO: Once write-behind is implemented send a contentstorefunc down to the create.
Func <IContentStore> writeThroughStore = null;
if (!string.IsNullOrWhiteSpace(cacheConfig.CacheName))
{
ServiceClientRpcConfiguration rpcConfiguration;
if (cacheConfig.GrpcPort != 0)
{
rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort);
}
else
{
var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName);
var portReader = factory.GetPortReader();
var port = portReader.ReadPort();
rpcConfiguration = new ServiceClientRpcConfiguration(port);
}
writeThroughStore = () =>
new ServiceClientContentStore(
logger,
fileSystem,
cacheConfig.CacheName,
rpcConfiguration,
cacheConfig.ConnectionRetryIntervalSeconds,
cacheConfig.ConnectionRetryCount,
scenario: cacheConfig.ScenarioName);
}
BuildCacheServiceConfiguration buildCacheServiceConfiguration = cacheConfig.AsBuildCacheServiceConfigurationFile();
return(BuildCacheCacheFactory.Create(fileSystem, logger, credentialsFactory, buildCacheServiceConfiguration, writeThroughStore));
}
