private IList <ContentHashWithLastAccessTimeAndReplicaCount> GetUnpinnedHashesAndCompilePinnedSize( Context context, IEnumerable <ContentHashWithLastAccessTimeAndReplicaCount> hashesToPurge, PinnedSizeChecker pinnedSizeChecker, PurgeResult purgeResult) { long totalPinnedSize = 0; // Compile aggregate pinnedSize for the fail faster case var unpinnedHashes = new List <ContentHashWithLastAccessTimeAndReplicaCount>(); foreach (var hashInfo in hashesToPurge) { var pinnedSize = pinnedSizeChecker(context, hashInfo.ContentHash); if (pinnedSize >= 0) { totalPinnedSize += pinnedSize; } else { unpinnedHashes.Add(hashInfo); } } purgeResult.MergePinnedSize(totalPinnedSize); return(unpinnedHashes); }
private async Task <PurgeResult> EvictLocalAsync( Context context, IReadOnlyList <ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo, long reserveSize, CancellationToken cts) { var result = new PurgeResult(); foreach (var contentHashInfo in contentHashesWithInfo) { if (StopPurging(reserveSize, cts, out _)) { break; } var r = await _evictAsync(context, contentHashInfo, OnlyUnlinked); if (!r.Succeeded) { var errorResult = new PurgeResult(r); errorResult.Merge(result); return(errorResult); } result.Merge(r); } return(result); }
public PurgeResponse(PurgeResult result, IDictionary <CdnType, string[]> failedUrls, IEnumerable <string> failMessages, AggregateException exception) { Result = result; FailedUrls = failedUrls ?? new Dictionary <CdnType, string[]>(); FailMessages = failMessages ?? new string[0]; Exception = exception; }
public PurgeResponse(PurgeResult result, CdnType cdnType, IEnumerable <string> failedUrls, IEnumerable <string> failMessages, AggregateException exception) { Result = result; FailedUrls = failedUrls?.Any() ?? false ? new Dictionary <CdnType, string[]> { { cdnType, failedUrls.ToArray() } } : new Dictionary <CdnType, string[]>(); FailMessages = failMessages ?? new string[0]; Exception = exception; }
private Task <PurgeResult> Purge(Context context, long reserveSize) { return(PurgeCall.RunAsync(_tracer, new OperationContext(context), async() => { var result = new PurgeResult(); try { if (_token.IsCancellationRequested) { _tracer.Debug(context, "Purge exiting due to shutdown."); return result; } var purgableBytes = CurrentSize; if (_usePreviousPinnedBytes) { purgableBytes -= _previousPinnedBytes; } else { _previousPinnedBytes = 0; _usePreviousPinnedBytes = true; } var contentHashesWithLastAccessTime = await _store.GetLruOrderedContentListWithTimeAsync(); foreach (var rule in _rules) { if (_token.IsCancellationRequested) { _tracer.Debug(context, "Purge exiting due to shutdown."); return result; } var r = await rule.PurgeAsync(context, reserveSize, contentHashesWithLastAccessTime, _token); if (!r.Succeeded) { r.Merge(result); return r; } _previousPinnedBytes = Math.Max(_previousPinnedBytes, r.PinnedSize); result.Merge(r); } } catch (Exception exception) { _tracer.Warning(context, $"Purge threw exception: {exception}"); } result.CurrentContentSize = CurrentSize; return result; })); }
private PurgeResult RequestPurge(PurgeOptions options, string[] resources) { _logger.Info("Requesting Akamai Purge Resources"); PurgeResult result = null; using (var client = new PurgeApiClient()) { //result = (new PurgeResult { estTime = 0, resultMsg = "Completed Purge", resultCode = 100 }); result = client.purgeRequest(_config.Username, _config.Password, Network, options.Options, resources); } return(result); }
private void LogResultSuccess(PurgeResult result) { _view.Warn("Success Result Code " + result.resultCode); _view.Warn("Success Result Message " + result.resultMsg); if (result.modifiers != null) { foreach (var s in result.modifiers) { _view.Warn("Result Modifier " + s); } } _view.Warn("Result Est Time " + result.estTime); }
public Purger( Context context, QuotaKeeper quotaKeeper, DistributedEvictionSettings distributedEvictionSettings, IReadOnlyList <ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo, PurgeResult purgeResult, CancellationToken token) { _distributedEvictionSettings = distributedEvictionSettings; _context = context; _quotaKeeper = quotaKeeper; _contentHashesWithInfo = contentHashesWithInfo; _purgeResult = purgeResult; _token = token; }
public Purger( Context context, QuotaKeeper quotaKeeper, IDistributedLocationStore?distributedStore, IReadOnlyList <ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithInfo, PurgeResult purgeResult, CancellationToken token) { _context = context; _quotaKeeper = quotaKeeper; _distributedStore = distributedStore; _contentHashesWithInfo = contentHashesWithInfo; _purgeResult = purgeResult; _token = token; }
/// <summary> /// Evict hashes in LRU-ed order determined by remote last-access times. /// </summary> /// <param name="context">Context.</param> /// <param name="hashesToPurge">Hashes in LRU-ed order based on local last-access time.</param> /// <param name="reserveSize">Reserve size.</param> /// <param name="cts">Cancellation token source.</param> private async Task <PurgeResult> EvictDistributedWithDistributedStoreAsync( Context context, IReadOnlyList <ContentHashWithLastAccessTimeAndReplicaCount> hashesToPurge, long reserveSize, CancellationToken cts) { var result = new PurgeResult(reserveSize, hashesToPurge.Count, _quota.ToString()); var evictedContent = new List <ContentHash>(); var distributedStore = _distributedEvictionSettings.DistributedStore; foreach (var contentHashInfo in distributedStore.GetLruPages(context, hashesToPurge).SelectMany(p => p)) { if (StopPurging(reserveSize, cts, out var stopReason)) { result.StopReason = stopReason; break; } var r = await _evictAsync(context, contentHashInfo, OnlyUnlinked); if (!r.Succeeded) { var errorResult = new PurgeResult(r); errorResult.Merge(result); return(errorResult); } if (r.SuccessfullyEvictedHash) { evictedContent.Add(contentHashInfo.ContentHash); } result.Merge(r); } var unregisterResult = await distributedStore.UnregisterAsync(context, evictedContent, cts); if (!unregisterResult) { var errorResult = new PurgeResult(unregisterResult); errorResult.Merge(result); return(errorResult); } return(result); }
private async Task Reserve(Context context) { context.Debug($"Starting purge loop. Current Size={CurrentSize}"); // GetConsumingEnumerable means that this loop will block whenever the ReserveQueue is empty, // and then the loop will complete once ReserveQueue.CompleteAdding() is called and the // ReserveQueue empties. foreach (var request in _reserveQueue.GetConsumingEnumerable()) { try { if (request.Value.IsCalibrateRequest) { foreach (var rule in _rules.Where(r => r.CanBeCalibrated)) { await CalibrateAsync(context, rule); } continue; } Contract.Assert(request.Value.IsReserveRequest); var reserveSize = request.Value.Size; var reserved = false; long requestIdStartedPurge = 0; var failIfDoesNotFit = false; PurgeResult purgeResult = null; Action reserve = () => { Interlocked.Add(ref _size, reserveSize); reserved = true; reserveSize = 0; }; do { // Shutdown started while a request call was ongoing, so we'll throw for the requesting caller. if (_token.IsCancellationRequested) { const string message = "Reserve exiting due to shutdown"; _tracer.Debug(context, message); request.Complete(message); break; } // If space is immediately available under the hard limit, reserve it and complete the request. var rulesNotInsideHardLimit = _rules.Where(rule => !rule.IsInsideHardLimit(reserveSize).Succeeded).ToList(); if (rulesNotInsideHardLimit.Count == 0) { reserve(); } else if (failIfDoesNotFit) { // Purge task has finished here. Contract.Assert(_purgeTask == null); var rulesCannotBeCalibratedResults = rulesNotInsideHardLimit.Where(rule => !rule.CanBeCalibrated) .Select(rule => rule.IsInsideHardLimit(reserveSize)) .ToList(); if (rulesCannotBeCalibratedResults.Any()) { // Some rule has reached its hard limit, and its quota cannot be calibrated. var sb = new StringBuilder(); sb.AppendLine("Error: Failed to make space."); foreach (var ruleResult in rulesCannotBeCalibratedResults) { sb.AppendLine($"Hard limit surpassed. {ruleResult.ErrorMessage}"); } request.Complete(sb.ToString()); break; } // All rules that reached their hard limits can be calibrated. We will disable such rules temporarily until calibration. foreach (var rule in rulesNotInsideHardLimit.Where(rule => rule.CanBeCalibrated)) { rule.IsEnabled = false; } reserve(); } // Start purge if not already running and if now over the soft limit. if (_purgeTask == null) { var softLimitResult = _rules.Select(rule => rule.IsInsideSoftLimit(reserveSize)).ToList(); if (!softLimitResult.All(rule => rule.Succeeded)) { foreach (var ruleResult in softLimitResult.Where(rule => !rule.Succeeded)) { _tracer.Debug(context, $"Soft limit surpassed - Purge started. {ruleResult.ErrorMessage}"); } requestIdStartedPurge = request.Id; _purgeTask = Task.Run(() => Purge(context, reserveSize)); } } // Complete request if reserved. Do this only after starting the purge task just above // so that any immediate Sync call does not race with starting of that task. if (reserved) { request.Complete(null); } // If the current request has not yet been satisfied, wait until either the purge task completes or some // content is evicted before trying again. If the purge task completes, set it to null so the next iteration // of the loop will restart it. if (!reserved && _purgeTask == await Task.WhenAny(_purgeTask, _contentItemEvicted.WaitAsync())) { try { purgeResult = await _purgeTask; // If the request that initiated this completed purge is still waiting, then it was not unblocked // by any intermediate contentItemEvicted events. Cause request to fail if next check for fit // fails. Otherwise, the loop will run again. if (request.Id == requestIdStartedPurge) { failIfDoesNotFit = true; } } finally { _purgeTask = null; } } }while (!reserved && (purgeResult == null || purgeResult.Succeeded || _rules.Any(r => r.CanBeCalibrated))); if (!reserved && purgeResult != null && !purgeResult.Succeeded) { request.Complete(purgeResult.ErrorMessage); } } catch (Exception e) { // Any unexpected exception (usually from the purge task) means reservation failed. var message = $"Reservation failed for size=[{request.Value}]: {e}"; _tracer.Warning(context, message); request.Complete(message); } } }
private void LogResultFailure(PurgeResult result) { _view.Error("Error Result Code " + result.resultCode); _view.Error("Error Result Message " + result.resultMsg); }
private async Task <(bool finishedPurging, PurgeResult purgeResult)> ProcessHashesForEvictionAsync( IList <ContentHashWithLastAccessTimeAndReplicaCount> contentHashesWithRemoteInfo, long reserveSize, CancellationToken cts, Context context, PurgeResult finalResult, Dictionary <ContentHash, Tuple <bool, DateTime> > unpurgedHashes, PriorityQueue <ContentHashWithLastAccessTimeAndReplicaCount> hashQueue, List <ContentHash> evictedHashes) { bool finishedPurging = false; foreach (var contentHashWithRemoteInfo in contentHashesWithRemoteInfo) { if (StopPurging(reserveSize, cts, out _)) { finishedPurging = true; } var trackHash = true; if (!finishedPurging) { // If not done purging and locations is negative, safe to evict immediately because contentHash has either: // 1) Aged out of content tracker // 2) Has matching last-access time (implying that the hash's last-access time is in sync with the datacenter) if (contentHashWithRemoteInfo.SafeToEvict) { var evictResult = await _evictAsync( context, contentHashWithRemoteInfo, OnlyUnlinked); if (!evictResult.Succeeded) { var errorResult = new PurgeResult(evictResult); errorResult.Merge(finalResult); return(finishedPurging, errorResult); } finalResult.Merge(evictResult); // SLIGHT HACK: Only want to keep track of hashes that unsuccessfully evicted and were unpinned at eviction time. // We can determine that it was unpinned by PinnedSize, which is pinned bytes encountered during eviction. trackHash = !evictResult.SuccessfullyEvictedHash && evictResult.PinnedSize == 0; } else { hashQueue.Push(contentHashWithRemoteInfo); } } if (trackHash) { unpurgedHashes[contentHashWithRemoteInfo.ContentHash] = Tuple.Create(contentHashWithRemoteInfo.SafeToEvict, contentHashWithRemoteInfo.LastAccessTime); } else { // Don't track hash to update with remote last-access time because it was evicted unpurgedHashes.Remove(contentHashWithRemoteInfo.ContentHash); evictedHashes.Add(contentHashWithRemoteInfo.ContentHash); } } return(finishedPurging, (PurgeResult)null); }
private static void AssertNoPurging(PurgeResult result) { result.EvictedFiles.Should().Be(0); result.EvictedSize.Should().Be(0); result.PinnedSize.Should().Be(0); }
/// <summary> /// Attempts to evict hashes in hashesToPurge until the reserveSize is met or all hashes with in-sync local and remote last-access times have been evicted. /// </summary> /// <param name="context">Context.</param> /// <param name="hashesToPurge">Hashes sorted in LRU-ed order.</param> /// <param name="cts">Cancellation token source.</param> /// <param name="reserveSize">Reserve size.</param> /// <param name="unpurgedHashes">Hashes that were checked in the data center but not evicted.</param> /// <param name="evictedHashes">list of evicted hashes</param> private async Task <(bool finishedPurging, PurgeResult purgeResult)> AttemptPurgeAsync( Context context, IReadOnlyList <ContentHashWithLastAccessTimeAndReplicaCount> hashesToPurge, CancellationToken cts, long reserveSize, Dictionary <ContentHash, Tuple <bool, DateTime> > unpurgedHashes, List <ContentHash> evictedHashes) { var finalResult = new PurgeResult(); var finishedPurging = false; var trimOrGetLastAccessTimeAsync = _distributedEvictionSettings.TrimOrGetLastAccessTimeAsync; var batchSize = _distributedEvictionSettings.LocationStoreBatchSize; var pinAndSizeChecker = _distributedEvictionSettings.PinnedSizeChecker; var replicaCreditInMinutes = _distributedEvictionSettings.ReplicaCreditInMinutes; var hashQueue = CreatePriorityQueue(hashesToPurge, replicaCreditInMinutes); while (!finishedPurging && hashQueue.Count > 0) { var contentHashesWithInfo = GetLruBatch(hashQueue, batchSize); var unpinnedHashes = GetUnpinnedHashesAndCompilePinnedSize(context, contentHashesWithInfo, pinAndSizeChecker, finalResult); if (!unpinnedHashes.Any()) { continue; // No hashes in this batch are able to evicted because they're all pinned } // If unpurgedHashes contains hash, it was checked once in the data center. We relax the replica restriction on retry var unpinnedHashesWithCheck = unpinnedHashes.Select(hash => Tuple.Create(hash, !unpurgedHashes.ContainsKey(hash.ContentHash))).ToList(); // Unregister hashes that can be safely evicted and get distributed last-access time for the rest var contentHashesInfoRemoteResult = await trimOrGetLastAccessTimeAsync(context, unpinnedHashesWithCheck, cts, UrgencyHint.High); if (!contentHashesInfoRemoteResult.Succeeded) { var errorResult = new PurgeResult(contentHashesInfoRemoteResult); errorResult.Merge(finalResult); return(finishedPurging, errorResult); } var purgeInfo = await ProcessHashesForEvictionAsync( contentHashesInfoRemoteResult.Data, reserveSize, cts, context, finalResult, unpurgedHashes, hashQueue, evictedHashes); if (purgeInfo.purgeResult != null) { return(purgeInfo); // Purging encountered an error. } finishedPurging = purgeInfo.finishedPurging; } return(finishedPurging, finalResult); }
private static void AssertPurged(PurgeResult result, EvictResult expectedResult) { result.EvictedFiles.Should().Be(expectedResult.EvictedFiles); result.EvictedSize.Should().Be(expectedResult.EvictedSize); result.PinnedSize.Should().Be(expectedResult.PinnedSize); }
public void Aggregate_GivenMultipleResults_ThenPrioritise(PurgeResult[] mockResults, PurgeResult expectedResult) { var subjects = mockResults.Select(r => new PurgeResponse(r, null, null, null)).ToArray(); var result = PurgeResponse.Aggregate(subjects); Assert.AreEqual(expectedResult, result.Result); }