protected async Task <IOperationResult> ExecuteOperation(string collectionName, long start, long take, CollectionOperationOptions options, DocumentsOperationContext context,
Action <DeterminateProgress> onProgress, Func <string, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token)
{
var progress = new DeterminateProgress();
var cancellationToken = token.Token;
var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection;
long lastEtag;
long totalCount;
using (context.OpenReadTransaction())
{
lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs);
totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs);
}
progress.Total = totalCount;
// send initial progress with total count set, and 0 as processed count
onProgress(progress);
long startEtag = 0;
var alreadySeenIdsCount = new Reference <long>();
string startAfterId = null;
using (var rateGate = options.MaxOpsPerSecond.HasValue
? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1))
: null)
{
var end = false;
var ids = new Queue <string>(OperationBatchSize);
while (startEtag <= lastEtag)
{
cancellationToken.ThrowIfCancellationRequested();
ids.Clear();
Database.ForTestingPurposes?.CollectionRunnerBeforeOpenReadTransaction?.Invoke();
using (context.OpenReadTransaction())
{
foreach (var document in GetDocuments(context, collectionName, startEtag, startAfterId, alreadySeenIdsCount, OperationBatchSize, isAllDocs, DocumentFields.Id, out bool isStartsWithOrIdQuery))
{
using (document)
{
cancellationToken.ThrowIfCancellationRequested();
token.Delay();
if (isAllDocs && document.Id.StartsWith(HiLoHandler.RavenHiloIdPrefix, StringComparison.OrdinalIgnoreCase))
{
continue;
}
// start with and id queries aren't ordered by the etag
if (isStartsWithOrIdQuery == false && document.Etag > lastEtag)
{
// we don't want to go over the documents that we have patched
end = true;
break;
}
startEtag = document.Etag + 1;
if (start > 0)
{
start--;
continue;
}
if (take-- <= 0)
{
end = true;
break;
}
startAfterId = document.Id;
ids.Enqueue(document.Id);
context.Transaction.ForgetAbout(document);
}
}
}
if (ids.Count == 0)
{
break;
}
do
{
var command = new ExecuteRateLimitedOperations <string>(ids, action, rateGate, token,
maxTransactionSize: 16 * Voron.Global.Constants.Size.Megabyte,
batchSize: OperationBatchSize);
await Database.TxMerger.Enqueue(command);
progress.Processed += command.Processed;
onProgress(progress);
if (command.NeedWait)
{
rateGate?.WaitToProceed();
}
token.Delay();
} while (ids.Count > 0);
if (end)
{
break;
}
}
}
return(new BulkOperationResult
{
Total = progress.Processed
});
}
protected async Task <IOperationResult> ExecuteOperation(string collectionName, CollectionOperationOptions options, DocumentsOperationContext context,
Action <DeterminateProgress> onProgress, Func <LazyStringValue, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token)
{
const int batchSize = 1024;
var progress = new DeterminateProgress();
var cancellationToken = token.Token;
var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection;
long lastEtag;
long totalCount;
using (context.OpenReadTransaction())
{
lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs);
totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs);
}
progress.Total = totalCount;
// send initial progress with total count set, and 0 as processed count
onProgress(progress);
long startEtag = 0;
using (var rateGate = options.MaxOpsPerSecond.HasValue
? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1))
: null)
{
var end = false;
while (startEtag <= lastEtag)
{
cancellationToken.ThrowIfCancellationRequested();
using (context.OpenReadTransaction())
{
var ids = new Queue <LazyStringValue>(batchSize);
foreach (var document in GetDocuments(context, collectionName, startEtag, batchSize, isAllDocs))
{
cancellationToken.ThrowIfCancellationRequested();
token.Delay();
if (isAllDocs && IsHiLoDocument(document))
{
continue;
}
if (document.Etag > lastEtag) // we don't want to go over the documents that we have patched
{
end = true;
break;
}
startEtag = document.Etag + 1;
ids.Enqueue(document.Id);
}
if (ids.Count == 0)
{
break;
}
do
{
var command = new ExecuteRateLimitedOperations <LazyStringValue>(ids, action, rateGate, token);
await Database.TxMerger.Enqueue(command);
progress.Processed += command.Processed;
onProgress(progress);
if (command.NeedWait)
{
rateGate?.WaitToProceed();
}
} while (ids.Count > 0);
if (end)
{
break;
}
}
}
}
return(new BulkOperationResult
{
Total = progress.Processed
});
}