private async Task CleanupDocs(int batchSize, bool forExpiration)
{
var currentTime = _database.Time.GetUtcNow();
try
{
DatabaseTopology topology;
using (_database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverContext))
using (serverContext.OpenReadTransaction())
{
topology = _database.ServerStore.Cluster.ReadDatabaseTopology(serverContext, _database.Name);
}
var isFirstInTopology = string.Equals(topology.AllNodes.FirstOrDefault(), _database.ServerStore.NodeTag, StringComparison.OrdinalIgnoreCase);
using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
while (true)
{
context.Reset();
context.Renew();
using (context.OpenReadTransaction())
{
var options = new ExpirationStorage.ExpiredDocumentsOptions(context, currentTime, isFirstInTopology, batchSize);
var expired =
forExpiration ?
_database.DocumentsStorage.ExpirationStorage.GetExpiredDocuments(options, out var duration, CancellationToken) :
_database.DocumentsStorage.ExpirationStorage.GetDocumentsToRefresh(options, out duration, CancellationToken);
if (expired == null || expired.Count == 0)
{
return;
}
var command = new DeleteExpiredDocumentsCommand(expired, _database, forExpiration, currentTime);
await _database.TxMerger.Enqueue(command);
if (Logger.IsInfoEnabled)
{
Logger.Info($"Successfully {(forExpiration ? "deleted" : "refreshed")} {command.DeletionCount:#,#;;0} documents in {duration.ElapsedMilliseconds:#,#;;0} ms.");
}
}
}
}
}
catch (OperationCanceledException)
{
// this will stop processing
throw;
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations($"Failed to {(forExpiration ? "delete" : "refresh")} documents on {_database.Name} which are older than {currentTime}", e);
}
}
}
internal async Task CleanupExpiredDocs()
{
var currentTime = _database.Time.GetUtcNow();
try
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Trying to find expired documents to delete");
}
using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
Stopwatch duration;
Dictionary <Slice, List <(Slice LowerId, LazyStringValue Id)> > expired;
using (context.OpenReadTransaction())
{
expired = _database.DocumentsStorage.ExpirationStorage.GetExpiredDocuments(context, currentTime, CancellationToken, out duration);
if (expired == null)
{
return;
}
}
var command = new DeleteExpiredDocumentsCommand(expired, _database);
await _database.TxMerger.Enqueue(command);
if (Logger.IsInfoEnabled)
{
Logger.Info($"Successfully deleted {command.DeletionCount:#,#;;0} documents in {duration.ElapsedMilliseconds:#,#;;0} ms.");
}
}
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations($"Failed to delete expired documents on {_database.Name} which are older than {currentTime}", e);
}
}
}
internal async Task CleanupExpiredDocs()
{
var currentTime = _database.Time.GetUtcNow();
var currentTicks = currentTime.Ticks;
try
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Trying to find expired documents to delete");
}
using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
using (var tx = context.OpenReadTransaction())
{
var expirationTree = tx.InnerTransaction.CreateTree(DocumentsByExpiration);
Dictionary <Slice, List <(Slice LowerId, LazyStringValue Id)> > expired;
Stopwatch duration;
using (var it = expirationTree.Iterate(false))
{
if (it.Seek(Slices.BeforeAllKeys) == false)
{
return;
}
expired = new Dictionary <Slice, List <(Slice LowerId, LazyStringValue Id)> >();
duration = Stopwatch.StartNew();
do
{
var entryTicks = it.CurrentKey.CreateReader().ReadBigEndianInt64();
if (entryTicks >= currentTicks)
{
break;
}
var ticksAsSlice = it.CurrentKey.Clone(tx.InnerTransaction.Allocator);
var expiredDocs = new List <(Slice LowerId, LazyStringValue Id)>();
expired.Add(ticksAsSlice, expiredDocs);
using (var multiIt = expirationTree.MultiRead(it.CurrentKey))
{
if (multiIt.Seek(Slices.BeforeAllKeys))
{
do
{
if (CancellationToken.IsCancellationRequested)
{
return;
}
var clonedId = multiIt.CurrentKey.Clone(tx.InnerTransaction.Allocator);
try
{
var document = _database.DocumentsStorage.Get(context, clonedId);
if (document == null)
{
expiredDocs.Add((clonedId, null));
continue;
}
if (HasExpired(document.Data, currentTime) == false)
{
continue;
}
expiredDocs.Add((clonedId, document.Id));
}
catch (DocumentConflictException)
{
LazyStringValue id = null;
var allExpired = true;
var conflicts = _database.DocumentsStorage.ConflictsStorage.GetConflictsFor(context, clonedId);
if (conflicts.Count == 0)
{
continue;
}
foreach (var conflict in conflicts)
{
id = conflict.Id;
if (HasExpired(conflict.Doc, currentTime))
{
continue;
}
allExpired = false;
break;
}
if (allExpired)
{
expiredDocs.Add((clonedId, id));
}
}
} while (multiIt.MoveNext());
}
}
} while (it.MoveNext());
}
var command = new DeleteExpiredDocumentsCommand(expired, _database, Logger);
await _database.TxMerger.Enqueue(command);
if (Logger.IsInfoEnabled)
{
Logger.Info($"Successfully deleted {command.DeletionCount:#,#;;0} documents in {duration.ElapsedMilliseconds:#,#;;0} ms.");
}
}
}
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations($"Failed to delete expired documents on {_database.Name} which are older than {currentTime}", e);
}
}
}
