/// <summary>
/// Initializes a new instance of the <see cref="BuildCacheCache"/> class.
/// </summary>
/// <param name="fileSystem">Filesystem used to read/write files.</param>
/// <param name="cacheNamespace">the namespace of the cache that is communicated with.</param>
/// <param name="buildCacheHttpClientFactory">Factory for creatign a backing BuildCache http client.</param>
/// <param name="backingContentStoreHttpClientFactory">Factory for creating a backing store http client.</param>
/// <param name="maxFingerprintSelectorsToFetch">Maximum number of selectors to enumerate.</param>
/// <param name="timeToKeepUnreferencedContent">Initial time-to-live for unreferenced content.</param>
/// <param name="minimumTimeToKeepContentHashLists">Minimum time-to-live for created or referenced ContentHashLists.</param>
/// <param name="rangeOfTimeToKeepContentHashLists">Range of time beyond the minimum for the time-to-live of created or referenced ContentHashLists.</param>
/// <param name="logger">A logger for tracing.</param>
/// <param name="fingerprintIncorporationEnabled">Feature flag to enable fingerprints incorporation on shutdown</param>
/// <param name="maxDegreeOfParallelismForIncorporateRequests">Throttle the number of fingerprints chunks sent in parallel</param>
/// <param name="maxFingerprintsPerIncorporateRequest">Max fingerprints allowed per chunk</param>
/// <param name="writeThroughContentStoreFunc">Optional write-through store to allow writing-behind to BlobStore</param>
/// <param name="sealUnbackedContentHashLists">If true, the client will attempt to seal any unbacked ContentHashLists that it sees.</param>
/// <param name="useBlobContentHashLists">use blob based content hash lists.</param>
/// <param name="downloadBlobsThroughBlobStore">If true, gets blobs through BlobStore. If false, gets blobs from the Azure Uri.</param>
/// <param name="useDedupStore">If true, gets content through DedupStore. If false, gets content from BlobStore.</param>
public BuildCacheCache(
IAbsFileSystem fileSystem,
string cacheNamespace,
IBuildCacheHttpClientFactory buildCacheHttpClientFactory,
IArtifactHttpClientFactory backingContentStoreHttpClientFactory,
int maxFingerprintSelectorsToFetch,
TimeSpan timeToKeepUnreferencedContent,
TimeSpan minimumTimeToKeepContentHashLists,
TimeSpan rangeOfTimeToKeepContentHashLists,
ILogger logger,
bool fingerprintIncorporationEnabled,
int maxDegreeOfParallelismForIncorporateRequests,
int maxFingerprintsPerIncorporateRequest,
Func <IContentStore> writeThroughContentStoreFunc = null,
bool sealUnbackedContentHashLists = false,
bool useBlobContentHashLists = false,
bool downloadBlobsThroughBlobStore = false,
bool useDedupStore = false)
{
Contract.Requires(fileSystem != null);
Contract.Requires(buildCacheHttpClientFactory != null);
Contract.Requires(backingContentStoreHttpClientFactory != null);
_fileSystem = fileSystem;
_cacheNamespace = cacheNamespace;
_buildCacheHttpClientFactory = buildCacheHttpClientFactory;
_tracer = new BuildCacheCacheTracer(logger, nameof(BuildCacheCache));
_backingContentStore = new BackingContentStore(
fileSystem, backingContentStoreHttpClientFactory, timeToKeepUnreferencedContent, _tracer.ContentSessionTracer, downloadBlobsThroughBlobStore, useDedupStore);
if (useDedupStore)
{
// Guaranteed content is only available for BlobSessions. (bug 144396)
_sealUnbackedContentHashLists = false;
// BlobBuildCacheHttpClient is incompatible with Dedup hashes. (bug 1458510)
_useBlobContentHashLists = false;
}
else
{
_sealUnbackedContentHashLists = sealUnbackedContentHashLists;
_useBlobContentHashLists = useBlobContentHashLists;
}
_maxFingerprintSelectorsToFetch = maxFingerprintSelectorsToFetch;
_minimumTimeToKeepContentHashLists = minimumTimeToKeepContentHashLists;
_rangeOfTimeToKeepContentHashLists = rangeOfTimeToKeepContentHashLists;
if (writeThroughContentStoreFunc != null)
{
_writeThroughContentStore = writeThroughContentStoreFunc();
Contract.Assert(_writeThroughContentStore != null);
}
_fingerprintIncorporationEnabled = fingerprintIncorporationEnabled;
_maxDegreeOfParallelismForIncorporateRequests = maxDegreeOfParallelismForIncorporateRequests;
_maxFingerprintsPerIncorporateRequest = maxFingerprintsPerIncorporateRequest;
}
/// <summary>
/// Initializes a new instance of the <see cref="BuildCacheSession"/> class.
/// </summary>
/// <param name="fileSystem">Filesystem used to read/write files.</param>
/// <param name="name">Session name.</param>
/// <param name="implicitPin">Policy determining whether or not content should be automatically pinned on adds or gets.</param>
/// <param name="cacheNamespace">The namespace of the cache being communicated with.</param>
/// <param name="cacheId">The id of the cache being communicated with.</param>
/// <param name="contentHashListAdapter">Backing BuildCache http client.</param>
/// <param name="backingContentSession">Backing BlobStore content session.</param>
/// <param name="maxFingerprintSelectorsToFetch">Maximum number of selectors to enumerate for a GetSelectors call.</param>
/// <param name="minimumTimeToKeepContentHashLists">Minimum time-to-live for created or referenced ContentHashLists.</param>
/// <param name="rangeOfTimeToKeepContentHashLists">Range of time beyond the minimum for the time-to-live of created or referenced ContentHashLists.</param>
/// <param name="fingerprintIncorporationEnabled">Feature flag to enable fingerprints incorporation</param>
/// <param name="maxDegreeOfParallelismForIncorporateRequests">Throttle the number of fingerprints chunks sent in parallel</param>
/// <param name="maxFingerprintsPerIncorporateRequest">Max fingerprints allowed per chunk</param>
/// <param name="writeThroughContentSession">Optional write-through session to allow writing-behind to BlobStore</param>
/// <param name="sealUnbackedContentHashLists">If true, the client will attempt to seal any unbacked ContentHashLists that it sees.</param>
/// <param name="overrideUnixFileAccessMode">If true, overrides default Unix file access modes when placing files.</param>
/// <param name="tracer">A tracer for logging calls</param>
/// <param name="enableEagerFingerprintIncorporation"><see cref="BuildCacheServiceConfiguration.EnableEagerFingerprintIncorporation"/></param>
/// <param name="inlineFingerprintIncorporationExpiry"><see cref="BuildCacheServiceConfiguration.InlineFingerprintIncorporationExpiryHours"/></param>
/// <param name="eagerFingerprintIncorporationInterval"><see cref="BuildCacheServiceConfiguration.EagerFingerprintIncorporationNagleIntervalMinutes"/></param>
/// <param name="eagerFingerprintIncorporationBatchSize"><see cref="BuildCacheServiceConfiguration.EagerFingerprintIncorporationNagleBatchSize"/></param>
/// <param name="manuallyExtendContentLifetime">Whether to manually extend content lifetime when doing incorporate calls</param>
/// <param name="forceUpdateOnAddContentHashList">Whether to force an update and ignore existing CHLs when adding.</param>
public BuildCacheSession(
IAbsFileSystem fileSystem,
string name,
ImplicitPin implicitPin,
string cacheNamespace,
Guid cacheId,
IContentHashListAdapter contentHashListAdapter,
IBackingContentSession backingContentSession,
int maxFingerprintSelectorsToFetch,
TimeSpan minimumTimeToKeepContentHashLists,
TimeSpan rangeOfTimeToKeepContentHashLists,
bool fingerprintIncorporationEnabled,
int maxDegreeOfParallelismForIncorporateRequests,
int maxFingerprintsPerIncorporateRequest,
IContentSession writeThroughContentSession,
bool sealUnbackedContentHashLists,
bool overrideUnixFileAccessMode,
BuildCacheCacheTracer tracer,
bool enableEagerFingerprintIncorporation,
TimeSpan inlineFingerprintIncorporationExpiry,
TimeSpan eagerFingerprintIncorporationInterval,
int eagerFingerprintIncorporationBatchSize,
bool manuallyExtendContentLifetime,
bool forceUpdateOnAddContentHashList)
: base(
fileSystem,
name,
implicitPin,
cacheNamespace,
cacheId,
contentHashListAdapter,
backingContentSession,
maxFingerprintSelectorsToFetch,
minimumTimeToKeepContentHashLists,
rangeOfTimeToKeepContentHashLists,
fingerprintIncorporationEnabled,
maxDegreeOfParallelismForIncorporateRequests,
maxFingerprintsPerIncorporateRequest,
writeThroughContentSession,
sealUnbackedContentHashLists,
overrideUnixFileAccessMode,
tracer,
enableEagerFingerprintIncorporation,
inlineFingerprintIncorporationExpiry,
eagerFingerprintIncorporationInterval,
eagerFingerprintIncorporationBatchSize,
manuallyExtendContentLifetime,
forceUpdateOnAddContentHashList)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="BuildCacheReadOnlySession"/> class.
/// </summary>
/// <param name="fileSystem">A interface to read/write files.</param>
/// <param name="name">Session name.</param>
/// <param name="implicitPin">Policy determining whether or not content should be automatically pinned on adds or gets.</param>
/// <param name="cacheNamespace">the namespace of the cache in VSTS</param>
/// <param name="cacheId">the guid of the cache in VSTS</param>
/// <param name="contentHashListAdapter">Backing BuildCache http client.</param>
/// <param name="backingContentSession">Backing content session.</param>
/// <param name="maxFingerprintSelectorsToFetch">Maximum number of selectors to enumerate.</param>
/// <param name="minimumTimeToKeepContentHashLists">Minimum time-to-live for created or referenced ContentHashLists.</param>
/// <param name="rangeOfTimeToKeepContentHashLists">Range of time beyond the minimum for the time-to-live of created or referenced ContentHashLists.</param>
/// <param name="fingerprintIncorporationEnabled">Feature flag to enable fingerprints incorporation</param>
/// <param name="maxDegreeOfParallelismForIncorporateRequests">Throttle the number of fingerprints chunks sent in parallel</param>
/// <param name="maxFingerprintsPerIncorporateRequest">Max fingerprints allowed per chunk</param>
/// <param name="writeThroughContentSession">Optional write-through session to allow writing-behind to BlobStore</param>
/// <param name="sealUnbackedContentHashLists">If true, the client will attempt to seal any unbacked ContentHashLists that it sees.</param>
/// <param name="overrideUnixFileAccessMode">If true, overrides default Unix file access modes when placing files.</param>
/// <param name="tracer">A tracer for logging and perf counters.</param>
public BuildCacheReadOnlySession(
IAbsFileSystem fileSystem,
string name,
ImplicitPin implicitPin,
string cacheNamespace,
Guid cacheId,
IContentHashListAdapter contentHashListAdapter,
IContentSession backingContentSession,
int maxFingerprintSelectorsToFetch,
TimeSpan minimumTimeToKeepContentHashLists,
TimeSpan rangeOfTimeToKeepContentHashLists,
bool fingerprintIncorporationEnabled,
int maxDegreeOfParallelismForIncorporateRequests,
int maxFingerprintsPerIncorporateRequest,
IContentSession writeThroughContentSession,
bool sealUnbackedContentHashLists,
bool overrideUnixFileAccessMode,
BuildCacheCacheTracer tracer)
{
Contract.Requires(name != null);
Contract.Requires(contentHashListAdapter != null);
Contract.Requires(backingContentSession != null);
Contract.Requires(!backingContentSession.StartupStarted);
Name = name;
ImplicitPin = implicitPin;
ContentHashListAdapter = contentHashListAdapter;
BackingContentSession = backingContentSession;
_maxFingerprintSelectorsToFetch = maxFingerprintSelectorsToFetch;
CacheNamespace = cacheNamespace;
CacheId = cacheId;
WriteThroughContentSession = writeThroughContentSession;
_sealUnbackedContentHashLists = sealUnbackedContentHashLists;
Tracer = tracer;
_tempDirectory = new DisposableDirectory(fileSystem);
_sealingErrorsToPrintOnShutdown = new List <string>();
_fingerprintIncorporationEnabled = fingerprintIncorporationEnabled;
_maxDegreeOfParallelismForIncorporateRequests = maxDegreeOfParallelismForIncorporateRequests;
_maxFingerprintsPerIncorporateRequest = maxFingerprintsPerIncorporateRequest;
_overrideUnixFileAccessMode = overrideUnixFileAccessMode;
FingerprintTracker = new FingerprintTracker(DateTime.UtcNow + minimumTimeToKeepContentHashLists, rangeOfTimeToKeepContentHashLists);
}
/// <summary>
/// Initializes a new instance of the <see cref="BuildCacheSession"/> class.
/// </summary>
/// <param name="fileSystem">Filesystem used to read/write files.</param>
/// <param name="name">Session name.</param>
/// <param name="implicitPin">Policy determining whether or not content should be automatically pinned on adds or gets.</param>
/// <param name="cacheNamespace">The namespace of the cache being communicated with.</param>
/// <param name="cacheId">The id of the cache being communicated with.</param>
/// <param name="contentHashListAdapter">Backing BuildCache http client.</param>
/// <param name="backingContentSession">Backing BlobStore content session.</param>
/// <param name="maxFingerprintSelectorsToFetch">Maximum number of selectors to enumerate for a GetSelectors call.</param>
/// <param name="minimumTimeToKeepContentHashLists">Minimum time-to-live for created or referenced ContentHashLists.</param>
/// <param name="rangeOfTimeToKeepContentHashLists">Range of time beyond the minimum for the time-to-live of created or referenced ContentHashLists.</param>
/// <param name="fingerprintIncorporationEnabled">Feature flag to enable fingerprints incorporation</param>
/// <param name="maxDegreeOfParallelismForIncorporateRequests">Throttle the number of fingerprints chunks sent in parallel</param>
/// <param name="maxFingerprintsPerIncorporateRequest">Max fingerprints allowed per chunk</param>
/// <param name="writeThroughContentSession">Optional write-through session to allow writing-behind to BlobStore</param>
/// <param name="sealUnbackedContentHashLists">If true, the client will attempt to seal any unbacked ContentHashLists that it sees.</param>
/// <param name="overrideUnixFileAccessMode">If true, overrides default Unix file access modes when placing files.</param>
/// <param name="tracer">A tracer for logging calls</param>
public BuildCacheSession(
IAbsFileSystem fileSystem,
string name,
ImplicitPin implicitPin,
string cacheNamespace,
Guid cacheId,
IContentHashListAdapter contentHashListAdapter,
IContentSession backingContentSession,
int maxFingerprintSelectorsToFetch,
TimeSpan minimumTimeToKeepContentHashLists,
TimeSpan rangeOfTimeToKeepContentHashLists,
bool fingerprintIncorporationEnabled,
int maxDegreeOfParallelismForIncorporateRequests,
int maxFingerprintsPerIncorporateRequest,
IContentSession writeThroughContentSession,
bool sealUnbackedContentHashLists,
bool overrideUnixFileAccessMode,
BuildCacheCacheTracer tracer)
: base(
fileSystem,
name,
implicitPin,
cacheNamespace,
cacheId,
contentHashListAdapter,
backingContentSession,
maxFingerprintSelectorsToFetch,
minimumTimeToKeepContentHashLists,
rangeOfTimeToKeepContentHashLists,
fingerprintIncorporationEnabled,
maxDegreeOfParallelismForIncorporateRequests,
maxFingerprintsPerIncorporateRequest,
writeThroughContentSession,
sealUnbackedContentHashLists,
overrideUnixFileAccessMode,
tracer)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="BuildCacheCache"/> class.
/// </summary>
/// <param name="fileSystem">Filesystem used to read/write files.</param>
/// <param name="cacheNamespace">the namespace of the cache that is communicated with.</param>
/// <param name="buildCacheHttpClientFactory">Factory for creatign a backing BuildCache http client.</param>
/// <param name="backingContentStoreHttpClientFactory">Factory for creating a backing store http client.</param>
/// <param name="maxFingerprintSelectorsToFetch">Maximum number of selectors to enumerate.</param>
/// <param name="timeToKeepUnreferencedContent">Initial time-to-live for unreferenced content.</param>
/// <param name="pinInlineThreshold">Maximum time-to-live to inline pin calls.</param>
/// <param name="ignorePinThreshold">Minimum time-to-live to ignore pin calls.</param>
/// <param name="minimumTimeToKeepContentHashLists">Minimum time-to-live for created or referenced ContentHashLists.</param>
/// <param name="rangeOfTimeToKeepContentHashLists">Range of time beyond the minimum for the time-to-live of created or referenced ContentHashLists.</param>
/// <param name="logger">A logger for tracing.</param>
/// <param name="fingerprintIncorporationEnabled">Feature flag to enable fingerprints incorporation on shutdown</param>
/// <param name="maxDegreeOfParallelismForIncorporateRequests">Throttle the number of fingerprints chunks sent in parallel</param>
/// <param name="maxFingerprintsPerIncorporateRequest">Max fingerprints allowed per chunk</param>
/// <param name="domain">Domain ID to use against BlobStore or DedupStore</param>
/// <param name="writeThroughContentStoreFunc">Optional write-through store to allow writing-behind to BlobStore</param>
/// <param name="sealUnbackedContentHashLists">If true, the client will attempt to seal any unbacked ContentHashLists that it sees.</param>
/// <param name="useBlobContentHashLists">use blob based content hash lists.</param>
/// <param name="useDedupStore">If true, gets content through DedupStore. If false, gets content from BlobStore.</param>
/// <param name="overrideUnixFileAccessMode">If true, overrides default Unix file access modes.</param>
/// <param name="enableEagerFingerprintIncorporation"><see cref="BuildCacheServiceConfiguration.EnableEagerFingerprintIncorporation"/></param>
/// <param name="inlineFingerprintIncorporationExpiry"><see cref="BuildCacheServiceConfiguration.InlineFingerprintIncorporationExpiry"/></param>
/// <param name="eagerFingerprintIncorporationNagleInterval"><see cref="BuildCacheServiceConfiguration.EagerFingerprintIncorporationNagleInterval"/></param>
/// <param name="eagerFingerprintIncorporationNagleBatchSize"><see cref="BuildCacheServiceConfiguration.EagerFingerprintIncorporationNagleBatchSize"/></param>
/// <param name="downloadBlobsUsingHttpClient"><see cref="BuildCacheServiceConfiguration.DownloadBlobsUsingHttpClient"/></param>
public BuildCacheCache(
IAbsFileSystem fileSystem,
string cacheNamespace,
IBuildCacheHttpClientFactory buildCacheHttpClientFactory,
IArtifactHttpClientFactory backingContentStoreHttpClientFactory,
int maxFingerprintSelectorsToFetch,
TimeSpan timeToKeepUnreferencedContent,
TimeSpan pinInlineThreshold,
TimeSpan ignorePinThreshold,
TimeSpan minimumTimeToKeepContentHashLists,
TimeSpan rangeOfTimeToKeepContentHashLists,
ILogger logger,
bool fingerprintIncorporationEnabled,
int maxDegreeOfParallelismForIncorporateRequests,
int maxFingerprintsPerIncorporateRequest,
IDomainId domain,
Func <IContentStore> writeThroughContentStoreFunc = null,
bool sealUnbackedContentHashLists = false,
bool useBlobContentHashLists = false,
bool useDedupStore = false,
bool overrideUnixFileAccessMode = false,
bool enableEagerFingerprintIncorporation = false,
TimeSpan inlineFingerprintIncorporationExpiry = default,
TimeSpan eagerFingerprintIncorporationNagleInterval = default,
int eagerFingerprintIncorporationNagleBatchSize = 100,
bool downloadBlobsUsingHttpClient = false)
{
Contract.Requires(fileSystem != null);
Contract.Requires(buildCacheHttpClientFactory != null);
Contract.Requires(backingContentStoreHttpClientFactory != null);
_fileSystem = fileSystem;
_cacheNamespace = cacheNamespace;
_buildCacheHttpClientFactory = buildCacheHttpClientFactory;
_tracer = new BuildCacheCacheTracer(logger, nameof(BuildCacheCache));
_backingContentStore = new BackingContentStore(
new BackingContentStoreConfiguration()
{
FileSystem = fileSystem,
ArtifactHttpClientFactory = backingContentStoreHttpClientFactory,
TimeToKeepContent = timeToKeepUnreferencedContent,
PinInlineThreshold = pinInlineThreshold,
IgnorePinThreshold = ignorePinThreshold,
UseDedupStore = useDedupStore,
DownloadBlobsUsingHttpClient = downloadBlobsUsingHttpClient
});
_manuallyExtendContentLifetime = false;
if (useDedupStore)
{
// Guaranteed content is only available for BlobSessions. (bug 144396)
_sealUnbackedContentHashLists = false;
// BuildCache is incompatible with Dedup hashes.
// This is because BuildCache would not know to look for the blob in DedupStore instead of BlobStore
_useBlobContentHashLists = false;
_manuallyExtendContentLifetime = true;
}
else
{
_sealUnbackedContentHashLists = sealUnbackedContentHashLists;
_useBlobContentHashLists = useBlobContentHashLists;
}
if (!domain.Equals(WellKnownDomainIds.OriginalDomainId))
{
// BuildCache is incompatible with multi-domain
_useBlobContentHashLists = false;
_manuallyExtendContentLifetime = true;
}
_maxFingerprintSelectorsToFetch = maxFingerprintSelectorsToFetch;
_minimumTimeToKeepContentHashLists = minimumTimeToKeepContentHashLists;
_rangeOfTimeToKeepContentHashLists = rangeOfTimeToKeepContentHashLists;
if (writeThroughContentStoreFunc != null)
{
_writeThroughContentStore = writeThroughContentStoreFunc();
Contract.Assert(_writeThroughContentStore != null);
}
_fingerprintIncorporationEnabled = fingerprintIncorporationEnabled;
_maxDegreeOfParallelismForIncorporateRequests = maxDegreeOfParallelismForIncorporateRequests;
_maxFingerprintsPerIncorporateRequest = maxFingerprintsPerIncorporateRequest;
_overrideUnixFileAccessMode = overrideUnixFileAccessMode;
_enableEagerFingerprintIncorporation = enableEagerFingerprintIncorporation;
_inlineFingerprintIncorporationExpiry = inlineFingerprintIncorporationExpiry;
_eagerFingerprintIncorporationNagleInterval = eagerFingerprintIncorporationNagleInterval;
_eagerFingerprintIncorporationNagleBatchSize = eagerFingerprintIncorporationNagleBatchSize;
}