protected async Task <IOperationResult> ExecuteOperation(string collectionName, long start, long take, CollectionOperationOptions options, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, Func <string, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token) { var progress = new DeterminateProgress(); var cancellationToken = token.Token; var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection; long lastEtag; long totalCount; using (context.OpenReadTransaction()) { lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs); totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs); } progress.Total = totalCount; // send initial progress with total count set, and 0 as processed count onProgress(progress); long startEtag = 0; var alreadySeenIdsCount = new Reference <long>(); string startAfterId = null; using (var rateGate = options.MaxOpsPerSecond.HasValue ? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1)) : null) { var end = false; var ids = new Queue <string>(OperationBatchSize); while (startEtag <= lastEtag) { cancellationToken.ThrowIfCancellationRequested(); ids.Clear(); Database.ForTestingPurposes?.CollectionRunnerBeforeOpenReadTransaction?.Invoke(); using (context.OpenReadTransaction()) { foreach (var document in GetDocuments(context, collectionName, startEtag, startAfterId, alreadySeenIdsCount, OperationBatchSize, isAllDocs, DocumentFields.Id, out bool isStartsWithOrIdQuery)) { using (document) { cancellationToken.ThrowIfCancellationRequested(); token.Delay(); if (isAllDocs && document.Id.StartsWith(HiLoHandler.RavenHiloIdPrefix, StringComparison.OrdinalIgnoreCase)) { continue; } // start with and id queries aren't ordered by the etag if (isStartsWithOrIdQuery == false && document.Etag > lastEtag) { // we don't want to go over the documents that we have patched end = true; break; } startEtag = document.Etag + 1; if (start > 0) { start--; continue; } if (take-- <= 0) { end = true; break; } startAfterId = document.Id; ids.Enqueue(document.Id); context.Transaction.ForgetAbout(document); } } } if (ids.Count == 0) { break; } do { var command = new ExecuteRateLimitedOperations <string>(ids, action, rateGate, token, maxTransactionSize: 16 * Voron.Global.Constants.Size.Megabyte, batchSize: OperationBatchSize); await Database.TxMerger.Enqueue(command); progress.Processed += command.Processed; onProgress(progress); if (command.NeedWait) { rateGate?.WaitToProceed(); } token.Delay(); } while (ids.Count > 0); if (end) { break; } } } return(new BulkOperationResult { Total = progress.Processed }); }
protected async Task <IOperationResult> ExecuteOperation(string collectionName, CollectionOperationOptions options, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, Func <LazyStringValue, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token) { const int batchSize = 1024; var progress = new DeterminateProgress(); var cancellationToken = token.Token; var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection; long lastEtag; long totalCount; using (context.OpenReadTransaction()) { lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs); totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs); } progress.Total = totalCount; // send initial progress with total count set, and 0 as processed count onProgress(progress); long startEtag = 0; using (var rateGate = options.MaxOpsPerSecond.HasValue ? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1)) : null) { var end = false; while (startEtag <= lastEtag) { cancellationToken.ThrowIfCancellationRequested(); using (context.OpenReadTransaction()) { var ids = new Queue <LazyStringValue>(batchSize); foreach (var document in GetDocuments(context, collectionName, startEtag, batchSize, isAllDocs)) { cancellationToken.ThrowIfCancellationRequested(); token.Delay(); if (isAllDocs && IsHiLoDocument(document)) { continue; } if (document.Etag > lastEtag) // we don't want to go over the documents that we have patched { end = true; break; } startEtag = document.Etag + 1; ids.Enqueue(document.Id); } if (ids.Count == 0) { break; } do { var command = new ExecuteRateLimitedOperations <LazyStringValue>(ids, action, rateGate, token); await Database.TxMerger.Enqueue(command); progress.Processed += command.Processed; onProgress(progress); if (command.NeedWait) { rateGate?.WaitToProceed(); } } while (ids.Count > 0); if (end) { break; } } } } return(new BulkOperationResult { Total = progress.Processed }); }