/// <summary>
/// Initializes a new instance of the <see cref="BuildCacheReadOnlySession"/> class.
/// </summary>
/// <param name="fileSystem">A interface to read/write files.</param>
/// <param name="name">Session name.</param>
/// <param name="implicitPin">Policy determining whether or not content should be automatically pinned on adds or gets.</param>
/// <param name="cacheNamespace">the namespace of the cache in VSTS</param>
/// <param name="cacheId">the guid of the cache in VSTS</param>
/// <param name="contentHashListAdapter">Backing BuildCache http client.</param>
/// <param name="backingContentSession">Backing content session.</param>
/// <param name="maxFingerprintSelectorsToFetch">Maximum number of selectors to enumerate.</param>
/// <param name="minimumTimeToKeepContentHashLists">Minimum time-to-live for created or referenced ContentHashLists.</param>
/// <param name="rangeOfTimeToKeepContentHashLists">Range of time beyond the minimum for the time-to-live of created or referenced ContentHashLists.</param>
/// <param name="fingerprintIncorporationEnabled">Feature flag to enable fingerprints incorporation</param>
/// <param name="maxDegreeOfParallelismForIncorporateRequests">Throttle the number of fingerprints chunks sent in parallel</param>
/// <param name="maxFingerprintsPerIncorporateRequest">Max fingerprints allowed per chunk</param>
/// <param name="writeThroughContentSession">Optional write-through session to allow writing-behind to BlobStore</param>
/// <param name="sealUnbackedContentHashLists">If true, the client will attempt to seal any unbacked ContentHashLists that it sees.</param>
/// <param name="overrideUnixFileAccessMode">If true, overrides default Unix file access modes when placing files.</param>
/// <param name="tracer">A tracer for logging and perf counters.</param>
public BuildCacheReadOnlySession(
IAbsFileSystem fileSystem,
string name,
ImplicitPin implicitPin,
string cacheNamespace,
Guid cacheId,
IContentHashListAdapter contentHashListAdapter,
IContentSession backingContentSession,
int maxFingerprintSelectorsToFetch,
TimeSpan minimumTimeToKeepContentHashLists,
TimeSpan rangeOfTimeToKeepContentHashLists,
bool fingerprintIncorporationEnabled,
int maxDegreeOfParallelismForIncorporateRequests,
int maxFingerprintsPerIncorporateRequest,
IContentSession writeThroughContentSession,
bool sealUnbackedContentHashLists,
bool overrideUnixFileAccessMode,
BuildCacheCacheTracer tracer)
{
Contract.Requires(name != null);
Contract.Requires(contentHashListAdapter != null);
Contract.Requires(backingContentSession != null);
Contract.Requires(!backingContentSession.StartupStarted);
Name = name;
ImplicitPin = implicitPin;
ContentHashListAdapter = contentHashListAdapter;
BackingContentSession = backingContentSession;
_maxFingerprintSelectorsToFetch = maxFingerprintSelectorsToFetch;
CacheNamespace = cacheNamespace;
CacheId = cacheId;
WriteThroughContentSession = writeThroughContentSession;
_sealUnbackedContentHashLists = sealUnbackedContentHashLists;
Tracer = tracer;
_tempDirectory = new DisposableDirectory(fileSystem);
_sealingErrorsToPrintOnShutdown = new List <string>();
_fingerprintIncorporationEnabled = fingerprintIncorporationEnabled;
_maxDegreeOfParallelismForIncorporateRequests = maxDegreeOfParallelismForIncorporateRequests;
_maxFingerprintsPerIncorporateRequest = maxFingerprintsPerIncorporateRequest;
_overrideUnixFileAccessMode = overrideUnixFileAccessMode;
FingerprintTracker = new FingerprintTracker(DateTime.UtcNow + minimumTimeToKeepContentHashLists, rangeOfTimeToKeepContentHashLists);
}
/// <inheritdoc />
public Task <BoolResult> IncorporateStrongFingerprintsAsync(
Context context,
IEnumerable <Task <StrongFingerprint> > strongFingerprints,
CancellationToken cts,
UrgencyHint urgencyHint)
{
return(IncorporateStrongFingerprintsCall.RunAsync(Tracer.MemoizationStoreTracer, context, async() =>
{
// BuildCache remembers fingerprints for all content that has been fetched or added through it's APIs.
// However, the build may end up satisfying some fingerprints without using BuildCache. The build or
// other cache instances in the cache topology can track these "other" fingerprints and ask that the
// fingerprints be included in the cache session using the incoporate API. BuildCache will extend the
// expiration of the fingerprints and mapped content, as if they had just been published.
foreach (var strongFingerprintTask in strongFingerprints)
{
var strongFingerprint = await strongFingerprintTask.ConfigureAwait(false);
// The Incorporate API currently does allow passing the expiration, so we can't pass it here.
FingerprintTracker.Track(strongFingerprint);
}
return BoolResult.Success;
}));
}
/// <inheritdoc />
public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
Context context,
StrongFingerprint strongFingerprint,
ContentHashListWithDeterminism contentHashListWithDeterminism,
CancellationToken cts,
UrgencyHint urgencyHint)
{
return(new OperationContext(context, cts).PerformOperationAsync(
Tracer.MemoizationStoreTracer,
async() =>
{
// TODO: Split this out into separate implementations for WriteThrough vs. WriteBehind (bug 1365340)
if (WriteThroughContentSession == null)
{
ContentAvailabilityGuarantee guarantee =
ManuallyExtendContentLifetime
? ContentAvailabilityGuarantee.NoContentBackedByCache
: ContentAvailabilityGuarantee.AllContentBackedByCache;
return await AddOrGetContentHashListAsync(
context,
strongFingerprint,
contentHashListWithDeterminism,
guarantee).ConfigureAwait(false);
}
// Ensure that the content exists somewhere before trying to add
if (!await EnsureContentIsAvailableAsync(
context, contentHashListWithDeterminism.ContentHashList.Hashes, cts, urgencyHint).ConfigureAwait(false))
{
return new AddOrGetContentHashListResult(
"Referenced content must exist in the cache before a new content hash list is added.");
}
DateTime expirationUtc = FingerprintTracker.GenerateNewExpiration();
var valueToAdd = new ContentHashListWithCacheMetadata(
contentHashListWithDeterminism,
expirationUtc,
ContentAvailabilityGuarantee.NoContentBackedByCache);
DateTime?rawExpiration = null;
const int addLimit = 3;
for (int addAttempts = 0; addAttempts < addLimit; addAttempts++)
{
var debugString = $"Adding contentHashList=[{valueToAdd.ContentHashListWithDeterminism.ContentHashList}] " +
$"determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with " +
$"contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}], expirationUtc=[{expirationUtc}], forceUpdate=[{ForceUpdateOnAddContentHashList}]";
Tracer.Debug(context, debugString);
Result <ContentHashListWithCacheMetadata> responseObject =
await ContentHashListAdapter.AddContentHashListAsync(
context,
CacheNamespace,
strongFingerprint,
valueToAdd,
forceUpdate: ForceUpdateOnAddContentHashList).ConfigureAwait(false);
if (!responseObject.Succeeded)
{
return new AddOrGetContentHashListResult(responseObject);
}
ContentHashListWithCacheMetadata response = responseObject.Value;
var inconsistencyErrorMessage = CheckForResponseInconsistency(response);
if (inconsistencyErrorMessage != null)
{
return new AddOrGetContentHashListResult(inconsistencyErrorMessage);
}
rawExpiration = response.GetRawExpirationTimeUtc();
ContentHashList contentHashListToReturn =
UnpackContentHashListAfterAdd(contentHashListWithDeterminism.ContentHashList, response);
CacheDeterminism determinismToReturn = UnpackDeterminism(response, CacheId);
bool needToUpdateExistingValue = await CheckNeedToUpdateExistingValueAsync(
context,
response,
contentHashListToReturn,
cts,
urgencyHint).ConfigureAwait(false);
if (!needToUpdateExistingValue)
{
SealIfNecessaryAfterUnbackedAddOrGet(context, strongFingerprint, contentHashListWithDeterminism, response);
await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false);
return new AddOrGetContentHashListResult(
new ContentHashListWithDeterminism(contentHashListToReturn, determinismToReturn));
}
var hashOfExistingContentHashList = response.HashOfExistingContentHashList;
Tracer.Debug(context, $"Attempting to replace unbacked value with hash {hashOfExistingContentHashList.ToHex()}");
valueToAdd = new ContentHashListWithCacheMetadata(
contentHashListWithDeterminism,
expirationUtc,
ContentAvailabilityGuarantee.NoContentBackedByCache,
hashOfExistingContentHashList
);
}
Tracer.Warning(
context,
$"Lost the AddOrUpdate race {addLimit} times against unbacked values. Returning as though the add succeeded for now.");
await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false);
return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None));
},
traceOperationStarted: true,
extraStartMessage: $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}",
extraEndMessage: _ => $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}"));
}
