private async Task CleanupDocs(int batchSize, bool forExpiration) { var currentTime = _database.Time.GetUtcNow(); try { DatabaseTopology topology; using (_database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverContext)) using (serverContext.OpenReadTransaction()) { topology = _database.ServerStore.Cluster.ReadDatabaseTopology(serverContext, _database.Name); } var isFirstInTopology = string.Equals(topology.AllNodes.FirstOrDefault(), _database.ServerStore.NodeTag, StringComparison.OrdinalIgnoreCase); using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { while (true) { context.Reset(); context.Renew(); using (context.OpenReadTransaction()) { var options = new ExpirationStorage.ExpiredDocumentsOptions(context, currentTime, isFirstInTopology, batchSize); var expired = forExpiration ? _database.DocumentsStorage.ExpirationStorage.GetExpiredDocuments(options, out var duration, CancellationToken) : _database.DocumentsStorage.ExpirationStorage.GetDocumentsToRefresh(options, out duration, CancellationToken); if (expired == null || expired.Count == 0) { return; } var command = new DeleteExpiredDocumentsCommand(expired, _database, forExpiration, currentTime); await _database.TxMerger.Enqueue(command); if (Logger.IsInfoEnabled) { Logger.Info($"Successfully {(forExpiration ? "deleted" : "refreshed")} {command.DeletionCount:#,#;;0} documents in {duration.ElapsedMilliseconds:#,#;;0} ms."); } } } } } catch (OperationCanceledException) { // this will stop processing throw; } catch (Exception e) { if (Logger.IsOperationsEnabled) { Logger.Operations($"Failed to {(forExpiration ? "delete" : "refresh")} documents on {_database.Name} which are older than {currentTime}", e); } } }
internal async Task CleanupExpiredDocs() { var currentTime = _database.Time.GetUtcNow(); try { if (Logger.IsInfoEnabled) { Logger.Info("Trying to find expired documents to delete"); } using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { Stopwatch duration; Dictionary <Slice, List <(Slice LowerId, LazyStringValue Id)> > expired; using (context.OpenReadTransaction()) { expired = _database.DocumentsStorage.ExpirationStorage.GetExpiredDocuments(context, currentTime, CancellationToken, out duration); if (expired == null) { return; } } var command = new DeleteExpiredDocumentsCommand(expired, _database); await _database.TxMerger.Enqueue(command); if (Logger.IsInfoEnabled) { Logger.Info($"Successfully deleted {command.DeletionCount:#,#;;0} documents in {duration.ElapsedMilliseconds:#,#;;0} ms."); } } } catch (Exception e) { if (Logger.IsOperationsEnabled) { Logger.Operations($"Failed to delete expired documents on {_database.Name} which are older than {currentTime}", e); } } }
internal async Task CleanupExpiredDocs() { var currentTime = _database.Time.GetUtcNow(); var currentTicks = currentTime.Ticks; try { if (Logger.IsInfoEnabled) { Logger.Info("Trying to find expired documents to delete"); } using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { using (var tx = context.OpenReadTransaction()) { var expirationTree = tx.InnerTransaction.CreateTree(DocumentsByExpiration); Dictionary <Slice, List <(Slice LowerId, LazyStringValue Id)> > expired; Stopwatch duration; using (var it = expirationTree.Iterate(false)) { if (it.Seek(Slices.BeforeAllKeys) == false) { return; } expired = new Dictionary <Slice, List <(Slice LowerId, LazyStringValue Id)> >(); duration = Stopwatch.StartNew(); do { var entryTicks = it.CurrentKey.CreateReader().ReadBigEndianInt64(); if (entryTicks >= currentTicks) { break; } var ticksAsSlice = it.CurrentKey.Clone(tx.InnerTransaction.Allocator); var expiredDocs = new List <(Slice LowerId, LazyStringValue Id)>(); expired.Add(ticksAsSlice, expiredDocs); using (var multiIt = expirationTree.MultiRead(it.CurrentKey)) { if (multiIt.Seek(Slices.BeforeAllKeys)) { do { if (CancellationToken.IsCancellationRequested) { return; } var clonedId = multiIt.CurrentKey.Clone(tx.InnerTransaction.Allocator); try { var document = _database.DocumentsStorage.Get(context, clonedId); if (document == null) { expiredDocs.Add((clonedId, null)); continue; } if (HasExpired(document.Data, currentTime) == false) { continue; } expiredDocs.Add((clonedId, document.Id)); } catch (DocumentConflictException) { LazyStringValue id = null; var allExpired = true; var conflicts = _database.DocumentsStorage.ConflictsStorage.GetConflictsFor(context, clonedId); if (conflicts.Count == 0) { continue; } foreach (var conflict in conflicts) { id = conflict.Id; if (HasExpired(conflict.Doc, currentTime)) { continue; } allExpired = false; break; } if (allExpired) { expiredDocs.Add((clonedId, id)); } } } while (multiIt.MoveNext()); } } } while (it.MoveNext()); } var command = new DeleteExpiredDocumentsCommand(expired, _database, Logger); await _database.TxMerger.Enqueue(command); if (Logger.IsInfoEnabled) { Logger.Info($"Successfully deleted {command.DeletionCount:#,#;;0} documents in {duration.ElapsedMilliseconds:#,#;;0} ms."); } } } } catch (Exception e) { if (Logger.IsOperationsEnabled) { Logger.Operations($"Failed to delete expired documents on {_database.Name} which are older than {currentTime}", e); } } }