private async Task <PurgeResult> EvictLocalAsync( Context context, IReadOnlyList <ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo, long reserveSize, CancellationToken cts) { var result = new PurgeResult(); foreach (var contentHashInfo in contentHashesWithInfo) { if (StopPurging(reserveSize, cts, out _)) { break; } var r = await _evictAsync(context, contentHashInfo, OnlyUnlinked); if (!r.Succeeded) { var errorResult = new PurgeResult(r); errorResult.Merge(result); return(errorResult); } result.Merge(r); } return(result); }
private Task <PurgeResult> Purge(Context context, long reserveSize) { return(PurgeCall.RunAsync(_tracer, new OperationContext(context), async() => { var result = new PurgeResult(); try { if (_token.IsCancellationRequested) { _tracer.Debug(context, "Purge exiting due to shutdown."); return result; } var purgableBytes = CurrentSize; if (_usePreviousPinnedBytes) { purgableBytes -= _previousPinnedBytes; } else { _previousPinnedBytes = 0; _usePreviousPinnedBytes = true; } var contentHashesWithLastAccessTime = await _store.GetLruOrderedContentListWithTimeAsync(); foreach (var rule in _rules) { if (_token.IsCancellationRequested) { _tracer.Debug(context, "Purge exiting due to shutdown."); return result; } var r = await rule.PurgeAsync(context, reserveSize, contentHashesWithLastAccessTime, _token); if (!r.Succeeded) { r.Merge(result); return r; } _previousPinnedBytes = Math.Max(_previousPinnedBytes, r.PinnedSize); result.Merge(r); } } catch (Exception exception) { _tracer.Warning(context, $"Purge threw exception: {exception}"); } result.CurrentContentSize = CurrentSize; return result; })); }
private async Task <BoolResult> EvictLocalAsync() { foreach (var contentHashInfo in _contentHashesWithInfo) { if (StopPurging(out var stopReason, out var rule)) { _purgeResult.StopReason = stopReason; break; } var evictionResult = await _quotaKeeper.EvictContentAsync(_context, contentHashInfo, rule.GetOnlyUnlinked()); if (!evictionResult) { return(evictionResult); } _purgeResult.Merge(evictionResult); } return(BoolResult.Success); }
/// <summary> /// Evict hashes in LRU-ed order determined by remote last-access times. /// </summary> /// <param name="context">Context.</param> /// <param name="hashesToPurge">Hashes in LRU-ed order based on local last-access time.</param> /// <param name="reserveSize">Reserve size.</param> /// <param name="cts">Cancellation token source.</param> private async Task <PurgeResult> EvictDistributedWithDistributedStoreAsync( Context context, IReadOnlyList <ContentHashWithLastAccessTimeAndReplicaCount> hashesToPurge, long reserveSize, CancellationToken cts) { var result = new PurgeResult(reserveSize, hashesToPurge.Count, _quota.ToString()); var evictedContent = new List <ContentHash>(); var distributedStore = _distributedEvictionSettings.DistributedStore; foreach (var contentHashInfo in distributedStore.GetLruPages(context, hashesToPurge).SelectMany(p => p)) { if (StopPurging(reserveSize, cts, out var stopReason)) { result.StopReason = stopReason; break; } var r = await _evictAsync(context, contentHashInfo, OnlyUnlinked); if (!r.Succeeded) { var errorResult = new PurgeResult(r); errorResult.Merge(result); return(errorResult); } if (r.SuccessfullyEvictedHash) { evictedContent.Add(contentHashInfo.ContentHash); } result.Merge(r); } var unregisterResult = await distributedStore.UnregisterAsync(context, evictedContent, cts); if (!unregisterResult) { var errorResult = new PurgeResult(unregisterResult); errorResult.Merge(result); return(errorResult); } return(result); }
private async Task <(bool finishedPurging, PurgeResult purgeResult)> ProcessHashesForEvictionAsync( IList <ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithRemoteInfo, long reserveSize, CancellationToken cts, Context context, PurgeResult finalResult, Dictionary <ContentHash, Tuple <bool, DateTime> > unpurgedHashes, PriorityQueue <ContentHashWithLastAccessTimeAndReplicaCount> hashQueue, List <ContentHash> evictedHashes) { bool finishedPurging = false; foreach (var contentHashWithRemoteInfo in contentHashesWithRemoteInfo) { if (StopPurging(reserveSize, cts, out _)) { finishedPurging = true; } var trackHash = true; if (!finishedPurging) { // If not done purging and locations is negative, safe to evict immediately because contentHash has either: // 1) Aged out of content tracker // 2) Has matching last-access time (implying that the hash's last-access time is in sync with the datacenter) if (contentHashWithRemoteInfo.SafeToEvict) { var evictResult = await _evictAsync( context, contentHashWithRemoteInfo, OnlyUnlinked); if (!evictResult.Succeeded) { var errorResult = new PurgeResult(evictResult); errorResult.Merge(finalResult); return(finishedPurging, errorResult); } finalResult.Merge(evictResult); // SLIGHT HACK: Only want to keep track of hashes that unsuccessfully evicted and were unpinned at eviction time. // We can determine that it was unpinned by PinnedSize, which is pinned bytes encountered during eviction. trackHash = !evictResult.SuccessfullyEvictedHash && evictResult.PinnedSize == 0; } else { hashQueue.Push(contentHashWithRemoteInfo); } } if (trackHash) { unpurgedHashes[contentHashWithRemoteInfo.ContentHash] = Tuple.Create(contentHashWithRemoteInfo.SafeToEvict, contentHashWithRemoteInfo.LastAccessTime); } else { // Don't track hash to update with remote last-access time because it was evicted unpurgedHashes.Remove(contentHashWithRemoteInfo.ContentHash); evictedHashes.Add(contentHashWithRemoteInfo.ContentHash); } } return(finishedPurging, (PurgeResult)null); }
/// <summary> /// Attempts to evict hashes in hashesToPurge until the reserveSize is met or all hashes with in-sync local and remote last-access times have been evicted. /// </summary> /// <param name="context">Context.</param> /// <param name="hashesToPurge">Hashes sorted in LRU-ed order.</param> /// <param name="cts">Cancellation token source.</param> /// <param name="reserveSize">Reserve size.</param> /// <param name="unpurgedHashes">Hashes that were checked in the data center but not evicted.</param> /// <param name="evictedHashes">list of evicted hashes</param> private async Task <(bool finishedPurging, PurgeResult purgeResult)> AttemptPurgeAsync( Context context, IReadOnlyList <ContentHashWithLastAccessTimeAndReplicaCount> hashesToPurge, CancellationToken cts, long reserveSize, Dictionary <ContentHash, Tuple <bool, DateTime> > unpurgedHashes, List <ContentHash> evictedHashes) { var finalResult = new PurgeResult(); var finishedPurging = false; var trimOrGetLastAccessTimeAsync = _distributedEvictionSettings.TrimOrGetLastAccessTimeAsync; var batchSize = _distributedEvictionSettings.LocationStoreBatchSize; var pinAndSizeChecker = _distributedEvictionSettings.PinnedSizeChecker; var replicaCreditInMinutes = _distributedEvictionSettings.ReplicaCreditInMinutes; var hashQueue = CreatePriorityQueue(hashesToPurge, replicaCreditInMinutes); while (!finishedPurging && hashQueue.Count > 0) { var contentHashesWithInfo = GetLruBatch(hashQueue, batchSize); var unpinnedHashes = GetUnpinnedHashesAndCompilePinnedSize(context, contentHashesWithInfo, pinAndSizeChecker, finalResult); if (!unpinnedHashes.Any()) { continue; // No hashes in this batch are able to evicted because they're all pinned } // If unpurgedHashes contains hash, it was checked once in the data center. We relax the replica restriction on retry var unpinnedHashesWithCheck = unpinnedHashes.Select(hash => Tuple.Create(hash, !unpurgedHashes.ContainsKey(hash.ContentHash))).ToList(); // Unregister hashes that can be safely evicted and get distributed last-access time for the rest var contentHashesInfoRemoteResult = await trimOrGetLastAccessTimeAsync(context, unpinnedHashesWithCheck, cts, UrgencyHint.High); if (!contentHashesInfoRemoteResult.Succeeded) { var errorResult = new PurgeResult(contentHashesInfoRemoteResult); errorResult.Merge(finalResult); return(finishedPurging, errorResult); } var purgeInfo = await ProcessHashesForEvictionAsync( contentHashesInfoRemoteResult.Data, reserveSize, cts, context, finalResult, unpurgedHashes, hashQueue, evictedHashes); if (purgeInfo.purgeResult != null) { return(purgeInfo); // Purging encountered an error. } finishedPurging = purgeInfo.finishedPurging; } return(finishedPurging, finalResult); }