public override long Execute(DocumentsOperationContext context, TransactionOperationsMerger.RecordingState recording)
{
var count = 0;
foreach (T id in _documentIds)
{
_cancellationToken.ThrowIfCancellationRequested();
_token.Delay();
if (_rateGate != null && _rateGate.WaitToProceed(0) == false)
{
NeedWait = true;
break;
}
count++;
var command = _commandToExecute(id);
try
{
Processed += command?.Execute(context, recording) ?? 0;
}
finally
{
if (command is IDisposable d)
{
d.Dispose();
}
}
if (_batchSize != null && Processed >= _batchSize)
{
break;
}
if (_maxTransactionSizeInPages != null &&
context.Transaction.InnerTransaction.LowLevelTransaction.NumberOfModifiedPages +
context.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes) / Constants.Storage.PageSize > _maxTransactionSizeInPages)
{
break;
}
}
var tx = context.Transaction.InnerTransaction.LowLevelTransaction;
tx.OnDispose += _ =>
{
if (tx.Committed == false)
{
return;
}
for (int i = 0; i < count; i++)
{
_documentIds.Dequeue();
}
};
return(Processed);
}
public override void AddResult(Document result)
{
if (_anyWrites == false)
{
StartResponseIfNeeded();
}
_writer.AddResult(result);
_token.Delay();
}
public void WillThrowWhenDelayingInfiniteTimeout()
{
using (var token = new OperationCancelToken(CancellationToken.None))
{
Assert.Throws <InvalidOperationException>(() => token.Delay());
}
using (var token = new OperationCancelToken(TimeSpan.FromMilliseconds(-1), CancellationToken.None))
{
Assert.Throws <InvalidOperationException>(() => token.Delay());
}
}
public override void AddResult(Document result)
{
using (result)
{
_token.Delay();
DocumentIds.Enqueue(result.Id);
_progress.Total++;
if (_progress.Total % 10_000 == 0)
{
_onProgress.Invoke(_progress);
}
}
}
public override ValueTask AddResultAsync(Document result, CancellationToken token)
{
using (result)
{
_token.Delay();
DocumentIds.Enqueue(result.Id);
_progress.Total++;
if (_progress.Total % 10_000 == 0)
{
_onProgress.Invoke(_progress);
}
}
return(default);
public override int Execute(DocumentsOperationContext context, TransactionOperationsMerger.RecordingState recording)
{
while (_documentIds.Count > 0)
{
_cancellationToken.ThrowIfCancellationRequested();
_token.Delay();
if (_rateGate != null && _rateGate.WaitToProceed(0) == false)
{
NeedWait = true;
break;
}
var id = _documentIds.Dequeue();
var command = _commandToExecute(id);
try
{
var count = command?.Execute(context, recording) ?? 0;
Processed += count;
}
finally
{
if (command is IDisposable d)
{
d.Dispose();
}
}
if (_batchSize != null && Processed >= _batchSize)
{
break;
}
if (_maxTransactionSizeInPages != null &&
context.Transaction.InnerTransaction.LowLevelTransaction.NumberOfModifiedPages +
context.Transaction.InnerTransaction.LowLevelTransaction.TotalEncryptionBufferSize.GetValue(SizeUnit.Bytes) / Constants.Storage.PageSize > _maxTransactionSizeInPages)
{
break;
}
}
return(Processed);
}
public override int Execute(DocumentsOperationContext context)
{
while (_documentIds.Count > 0)
{
_cancellationToken.ThrowIfCancellationRequested();
_token.Delay();
if (_rateGate != null && _rateGate.WaitToProceed(0) == false)
{
NeedWait = true;
break;
}
var id = _documentIds.Dequeue();
var command = _commandToExecute(id);
try
{
var count = command?.Execute(context) ?? 0;
Processed += count;
}
finally
{
if (command is IDisposable d)
{
d.Dispose();
}
}
if (_batchSize != null && Processed >= _batchSize)
{
break;
}
if (_maxTransactionSizeInPages != null &&
context.Transaction.InnerTransaction.LowLevelTransaction.NumberOfModifiedPages > _maxTransactionSizeInPages)
{
break;
}
}
return(Processed);
}
protected async Task <IOperationResult> ExecuteOperation(string collectionName, long start, long take, CollectionOperationOptions options, DocumentsOperationContext context,
Action <DeterminateProgress> onProgress, Func <string, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token)
{
var progress = new DeterminateProgress();
var cancellationToken = token.Token;
var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection;
long lastEtag;
long totalCount;
using (context.OpenReadTransaction())
{
lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs);
totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs);
}
progress.Total = totalCount;
// send initial progress with total count set, and 0 as processed count
onProgress(progress);
long startEtag = 0;
var alreadySeenIdsCount = new Reference <long>();
string startAfterId = null;
using (var rateGate = options.MaxOpsPerSecond.HasValue
? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1))
: null)
{
var end = false;
var ids = new Queue <string>(OperationBatchSize);
while (startEtag <= lastEtag)
{
cancellationToken.ThrowIfCancellationRequested();
ids.Clear();
Database.ForTestingPurposes?.CollectionRunnerBeforeOpenReadTransaction?.Invoke();
using (context.OpenReadTransaction())
{
foreach (var document in GetDocuments(context, collectionName, startEtag, startAfterId, alreadySeenIdsCount, OperationBatchSize, isAllDocs, DocumentFields.Id, out bool isStartsWithOrIdQuery))
{
using (document)
{
cancellationToken.ThrowIfCancellationRequested();
token.Delay();
if (isAllDocs && document.Id.StartsWith(HiLoHandler.RavenHiloIdPrefix, StringComparison.OrdinalIgnoreCase))
{
continue;
}
// start with and id queries aren't ordered by the etag
if (isStartsWithOrIdQuery == false && document.Etag > lastEtag)
{
// we don't want to go over the documents that we have patched
end = true;
break;
}
startEtag = document.Etag + 1;
if (start > 0)
{
start--;
continue;
}
if (take-- <= 0)
{
end = true;
break;
}
startAfterId = document.Id;
ids.Enqueue(document.Id);
context.Transaction.ForgetAbout(document);
}
}
}
if (ids.Count == 0)
{
break;
}
do
{
var command = new ExecuteRateLimitedOperations <string>(ids, action, rateGate, token,
maxTransactionSize: 16 * Voron.Global.Constants.Size.Megabyte,
batchSize: OperationBatchSize);
await Database.TxMerger.Enqueue(command);
progress.Processed += command.Processed;
onProgress(progress);
if (command.NeedWait)
{
rateGate?.WaitToProceed();
}
token.Delay();
} while (ids.Count > 0);
if (end)
{
break;
}
}
}
return(new BulkOperationResult
{
Total = progress.Processed
});
}
protected async Task <IOperationResult> ExecuteOperation(string collectionName, CollectionOperationOptions options, DocumentsOperationContext context,
Action <DeterminateProgress> onProgress, Func <LazyStringValue, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token)
{
const int batchSize = 1024;
var progress = new DeterminateProgress();
var cancellationToken = token.Token;
var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection;
long lastEtag;
long totalCount;
using (context.OpenReadTransaction())
{
lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs);
totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs);
}
progress.Total = totalCount;
// send initial progress with total count set, and 0 as processed count
onProgress(progress);
long startEtag = 0;
using (var rateGate = options.MaxOpsPerSecond.HasValue
? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1))
: null)
{
var end = false;
while (startEtag <= lastEtag)
{
cancellationToken.ThrowIfCancellationRequested();
using (context.OpenReadTransaction())
{
var ids = new Queue <LazyStringValue>(batchSize);
foreach (var document in GetDocuments(context, collectionName, startEtag, batchSize, isAllDocs))
{
cancellationToken.ThrowIfCancellationRequested();
token.Delay();
if (isAllDocs && IsHiLoDocument(document))
{
continue;
}
if (document.Etag > lastEtag) // we don't want to go over the documents that we have patched
{
end = true;
break;
}
startEtag = document.Etag + 1;
ids.Enqueue(document.Id);
}
if (ids.Count == 0)
{
break;
}
do
{
var command = new ExecuteRateLimitedOperations <LazyStringValue>(ids, action, rateGate, token);
await Database.TxMerger.Enqueue(command);
progress.Processed += command.Processed;
onProgress(progress);
if (command.NeedWait)
{
rateGate?.WaitToProceed();
}
} while (ids.Count > 0);
if (end)
{
break;
}
}
}
}
return(new BulkOperationResult
{
Total = progress.Processed
});
}
