private async Task <PurgeResult> EvictLocalAsync(
Context context,
IReadOnlyList <ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo,
long reserveSize,
CancellationToken cts)
{
var result = new PurgeResult();
foreach (var contentHashInfo in contentHashesWithInfo)
{
if (StopPurging(reserveSize, cts, out _))
{
break;
}
var r = await _evictAsync(context, contentHashInfo, OnlyUnlinked);
if (!r.Succeeded)
{
var errorResult = new PurgeResult(r);
errorResult.Merge(result);
return(errorResult);
}
result.Merge(r);
}
return(result);
}
private Task <PurgeResult> Purge(Context context, long reserveSize)
{
return(PurgeCall.RunAsync(_tracer, new OperationContext(context), async() =>
{
var result = new PurgeResult();
try
{
if (_token.IsCancellationRequested)
{
_tracer.Debug(context, "Purge exiting due to shutdown.");
return result;
}
var purgableBytes = CurrentSize;
if (_usePreviousPinnedBytes)
{
purgableBytes -= _previousPinnedBytes;
}
else
{
_previousPinnedBytes = 0;
_usePreviousPinnedBytes = true;
}
var contentHashesWithLastAccessTime = await _store.GetLruOrderedContentListWithTimeAsync();
foreach (var rule in _rules)
{
if (_token.IsCancellationRequested)
{
_tracer.Debug(context, "Purge exiting due to shutdown.");
return result;
}
var r = await rule.PurgeAsync(context, reserveSize, contentHashesWithLastAccessTime, _token);
if (!r.Succeeded)
{
r.Merge(result);
return r;
}
_previousPinnedBytes = Math.Max(_previousPinnedBytes, r.PinnedSize);
result.Merge(r);
}
}
catch (Exception exception)
{
_tracer.Warning(context, $"Purge threw exception: {exception}");
}
result.CurrentContentSize = CurrentSize;
return result;
}));
}
private async Task <BoolResult> EvictLocalAsync()
{
foreach (var contentHashInfo in _contentHashesWithInfo)
{
if (StopPurging(out var stopReason, out var rule))
{
_purgeResult.StopReason = stopReason;
break;
}
var evictionResult = await _quotaKeeper.EvictContentAsync(_context, contentHashInfo, rule.GetOnlyUnlinked());
if (!evictionResult)
{
return(evictionResult);
}
_purgeResult.Merge(evictionResult);
}
return(BoolResult.Success);
}
/// <summary>
/// Evict hashes in LRU-ed order determined by remote last-access times.
/// </summary>
/// <param name="context">Context.</param>
/// <param name="hashesToPurge">Hashes in LRU-ed order based on local last-access time.</param>
/// <param name="reserveSize">Reserve size.</param>
/// <param name="cts">Cancellation token source.</param>
private async Task <PurgeResult> EvictDistributedWithDistributedStoreAsync(
Context context,
IReadOnlyList <ContentHashWithLastAccessTimeAndReplicaCount> hashesToPurge,
long reserveSize,
CancellationToken cts)
{
var result = new PurgeResult(reserveSize, hashesToPurge.Count, _quota.ToString());
var evictedContent = new List <ContentHash>();
var distributedStore = _distributedEvictionSettings.DistributedStore;
foreach (var contentHashInfo in distributedStore.GetLruPages(context, hashesToPurge).SelectMany(p => p))
{
if (StopPurging(reserveSize, cts, out var stopReason))
{
result.StopReason = stopReason;
break;
}
var r = await _evictAsync(context, contentHashInfo, OnlyUnlinked);
if (!r.Succeeded)
{
var errorResult = new PurgeResult(r);
errorResult.Merge(result);
return(errorResult);
}
if (r.SuccessfullyEvictedHash)
{
evictedContent.Add(contentHashInfo.ContentHash);
}
result.Merge(r);
}
var unregisterResult = await distributedStore.UnregisterAsync(context, evictedContent, cts);
if (!unregisterResult)
{
var errorResult = new PurgeResult(unregisterResult);
errorResult.Merge(result);
return(errorResult);
}
return(result);
}
private async Task <(bool finishedPurging, PurgeResult purgeResult)> ProcessHashesForEvictionAsync(
IList <ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithRemoteInfo,
long reserveSize,
CancellationToken cts,
Context context,
PurgeResult finalResult,
Dictionary <ContentHash, Tuple <bool, DateTime> > unpurgedHashes,
PriorityQueue <ContentHashWithLastAccessTimeAndReplicaCount> hashQueue,
List <ContentHash> evictedHashes)
{
bool finishedPurging = false;
foreach (var contentHashWithRemoteInfo in contentHashesWithRemoteInfo)
{
if (StopPurging(reserveSize, cts, out _))
{
finishedPurging = true;
}
var trackHash = true;
if (!finishedPurging)
{
// If not done purging and locations is negative, safe to evict immediately because contentHash has either:
// 1) Aged out of content tracker
// 2) Has matching last-access time (implying that the hash's last-access time is in sync with the datacenter)
if (contentHashWithRemoteInfo.SafeToEvict)
{
var evictResult = await _evictAsync(
context,
contentHashWithRemoteInfo,
OnlyUnlinked);
if (!evictResult.Succeeded)
{
var errorResult = new PurgeResult(evictResult);
errorResult.Merge(finalResult);
return(finishedPurging, errorResult);
}
finalResult.Merge(evictResult);
// SLIGHT HACK: Only want to keep track of hashes that unsuccessfully evicted and were unpinned at eviction time.
// We can determine that it was unpinned by PinnedSize, which is pinned bytes encountered during eviction.
trackHash = !evictResult.SuccessfullyEvictedHash && evictResult.PinnedSize == 0;
}
else
{
hashQueue.Push(contentHashWithRemoteInfo);
}
}
if (trackHash)
{
unpurgedHashes[contentHashWithRemoteInfo.ContentHash] = Tuple.Create(contentHashWithRemoteInfo.SafeToEvict, contentHashWithRemoteInfo.LastAccessTime);
}
else
{
// Don't track hash to update with remote last-access time because it was evicted
unpurgedHashes.Remove(contentHashWithRemoteInfo.ContentHash);
evictedHashes.Add(contentHashWithRemoteInfo.ContentHash);
}
}
return(finishedPurging, (PurgeResult)null);
}
/// <summary>
/// Attempts to evict hashes in hashesToPurge until the reserveSize is met or all hashes with in-sync local and remote last-access times have been evicted.
/// </summary>
/// <param name="context">Context.</param>
/// <param name="hashesToPurge">Hashes sorted in LRU-ed order.</param>
/// <param name="cts">Cancellation token source.</param>
/// <param name="reserveSize">Reserve size.</param>
/// <param name="unpurgedHashes">Hashes that were checked in the data center but not evicted.</param>
/// <param name="evictedHashes">list of evicted hashes</param>
private async Task <(bool finishedPurging, PurgeResult purgeResult)> AttemptPurgeAsync(
Context context,
IReadOnlyList <ContentHashWithLastAccessTimeAndReplicaCount> hashesToPurge,
CancellationToken cts,
long reserveSize,
Dictionary <ContentHash, Tuple <bool, DateTime> > unpurgedHashes,
List <ContentHash> evictedHashes)
{
var finalResult = new PurgeResult();
var finishedPurging = false;
var trimOrGetLastAccessTimeAsync = _distributedEvictionSettings.TrimOrGetLastAccessTimeAsync;
var batchSize = _distributedEvictionSettings.LocationStoreBatchSize;
var pinAndSizeChecker = _distributedEvictionSettings.PinnedSizeChecker;
var replicaCreditInMinutes = _distributedEvictionSettings.ReplicaCreditInMinutes;
var hashQueue = CreatePriorityQueue(hashesToPurge, replicaCreditInMinutes);
while (!finishedPurging && hashQueue.Count > 0)
{
var contentHashesWithInfo = GetLruBatch(hashQueue, batchSize);
var unpinnedHashes = GetUnpinnedHashesAndCompilePinnedSize(context, contentHashesWithInfo, pinAndSizeChecker, finalResult);
if (!unpinnedHashes.Any())
{
continue; // No hashes in this batch are able to evicted because they're all pinned
}
// If unpurgedHashes contains hash, it was checked once in the data center. We relax the replica restriction on retry
var unpinnedHashesWithCheck = unpinnedHashes.Select(hash => Tuple.Create(hash, !unpurgedHashes.ContainsKey(hash.ContentHash))).ToList();
// Unregister hashes that can be safely evicted and get distributed last-access time for the rest
var contentHashesInfoRemoteResult =
await trimOrGetLastAccessTimeAsync(context, unpinnedHashesWithCheck, cts, UrgencyHint.High);
if (!contentHashesInfoRemoteResult.Succeeded)
{
var errorResult = new PurgeResult(contentHashesInfoRemoteResult);
errorResult.Merge(finalResult);
return(finishedPurging, errorResult);
}
var purgeInfo = await ProcessHashesForEvictionAsync(
contentHashesInfoRemoteResult.Data,
reserveSize,
cts,
context,
finalResult,
unpurgedHashes,
hashQueue,
evictedHashes);
if (purgeInfo.purgeResult != null)
{
return(purgeInfo); // Purging encountered an error.
}
finishedPurging = purgeInfo.finishedPurging;
}
return(finishedPurging, finalResult);
}
