public override long Execute(DocumentsOperationContext context, TransactionOperationsMerger.RecordingState recording) { var count = 0; foreach (T id in _documentIds) { _cancellationToken.ThrowIfCancellationRequested(); _token.Delay(); if (_rateGate != null && _rateGate.WaitToProceed(0) == false) { NeedWait = true; break; } count++; var command = _commandToExecute(id); try { Processed += command?.Execute(context, recording) ?? 0; } finally { if (command is IDisposable d) { d.Dispose(); } } if (_batchSize != null && Processed >= _batchSize) { break; } if (_maxTransactionSizeInPages != null && context.Transaction.InnerTransaction.LowLevelTransaction.NumberOfModifiedPages + context.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes) / Constants.Storage.PageSize > _maxTransactionSizeInPages) { break; } } var tx = context.Transaction.InnerTransaction.LowLevelTransaction; tx.OnDispose += _ => { if (tx.Committed == false) { return; } for (int i = 0; i < count; i++) { _documentIds.Dequeue(); } }; return(Processed); }
public override void AddResult(Document result) { if (_anyWrites == false) { StartResponseIfNeeded(); } _writer.AddResult(result); _token.Delay(); }
public void WillThrowWhenDelayingInfiniteTimeout() { using (var token = new OperationCancelToken(CancellationToken.None)) { Assert.Throws <InvalidOperationException>(() => token.Delay()); } using (var token = new OperationCancelToken(TimeSpan.FromMilliseconds(-1), CancellationToken.None)) { Assert.Throws <InvalidOperationException>(() => token.Delay()); } }
public override void AddResult(Document result) { using (result) { _token.Delay(); DocumentIds.Enqueue(result.Id); _progress.Total++; if (_progress.Total % 10_000 == 0) { _onProgress.Invoke(_progress); } } }
public override ValueTask AddResultAsync(Document result, CancellationToken token) { using (result) { _token.Delay(); DocumentIds.Enqueue(result.Id); _progress.Total++; if (_progress.Total % 10_000 == 0) { _onProgress.Invoke(_progress); } } return(default);
public override int Execute(DocumentsOperationContext context, TransactionOperationsMerger.RecordingState recording) { while (_documentIds.Count > 0) { _cancellationToken.ThrowIfCancellationRequested(); _token.Delay(); if (_rateGate != null && _rateGate.WaitToProceed(0) == false) { NeedWait = true; break; } var id = _documentIds.Dequeue(); var command = _commandToExecute(id); try { var count = command?.Execute(context, recording) ?? 0; Processed += count; } finally { if (command is IDisposable d) { d.Dispose(); } } if (_batchSize != null && Processed >= _batchSize) { break; } if (_maxTransactionSizeInPages != null && context.Transaction.InnerTransaction.LowLevelTransaction.NumberOfModifiedPages + context.Transaction.InnerTransaction.LowLevelTransaction.TotalEncryptionBufferSize.GetValue(SizeUnit.Bytes) / Constants.Storage.PageSize > _maxTransactionSizeInPages) { break; } } return(Processed); }
public override int Execute(DocumentsOperationContext context) { while (_documentIds.Count > 0) { _cancellationToken.ThrowIfCancellationRequested(); _token.Delay(); if (_rateGate != null && _rateGate.WaitToProceed(0) == false) { NeedWait = true; break; } var id = _documentIds.Dequeue(); var command = _commandToExecute(id); try { var count = command?.Execute(context) ?? 0; Processed += count; } finally { if (command is IDisposable d) { d.Dispose(); } } if (_batchSize != null && Processed >= _batchSize) { break; } if (_maxTransactionSizeInPages != null && context.Transaction.InnerTransaction.LowLevelTransaction.NumberOfModifiedPages > _maxTransactionSizeInPages) { break; } } return(Processed); }
protected async Task <IOperationResult> ExecuteOperation(string collectionName, long start, long take, CollectionOperationOptions options, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, Func <string, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token) { var progress = new DeterminateProgress(); var cancellationToken = token.Token; var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection; long lastEtag; long totalCount; using (context.OpenReadTransaction()) { lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs); totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs); } progress.Total = totalCount; // send initial progress with total count set, and 0 as processed count onProgress(progress); long startEtag = 0; var alreadySeenIdsCount = new Reference <long>(); string startAfterId = null; using (var rateGate = options.MaxOpsPerSecond.HasValue ? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1)) : null) { var end = false; var ids = new Queue <string>(OperationBatchSize); while (startEtag <= lastEtag) { cancellationToken.ThrowIfCancellationRequested(); ids.Clear(); Database.ForTestingPurposes?.CollectionRunnerBeforeOpenReadTransaction?.Invoke(); using (context.OpenReadTransaction()) { foreach (var document in GetDocuments(context, collectionName, startEtag, startAfterId, alreadySeenIdsCount, OperationBatchSize, isAllDocs, DocumentFields.Id, out bool isStartsWithOrIdQuery)) { using (document) { cancellationToken.ThrowIfCancellationRequested(); token.Delay(); if (isAllDocs && document.Id.StartsWith(HiLoHandler.RavenHiloIdPrefix, StringComparison.OrdinalIgnoreCase)) { continue; } // start with and id queries aren't ordered by the etag if (isStartsWithOrIdQuery == false && document.Etag > lastEtag) { // we don't want to go over the documents that we have patched end = true; break; } startEtag = document.Etag + 1; if (start > 0) { start--; continue; } if (take-- <= 0) { end = true; break; } startAfterId = document.Id; ids.Enqueue(document.Id); context.Transaction.ForgetAbout(document); } } } if (ids.Count == 0) { break; } do { var command = new ExecuteRateLimitedOperations <string>(ids, action, rateGate, token, maxTransactionSize: 16 * Voron.Global.Constants.Size.Megabyte, batchSize: OperationBatchSize); await Database.TxMerger.Enqueue(command); progress.Processed += command.Processed; onProgress(progress); if (command.NeedWait) { rateGate?.WaitToProceed(); } token.Delay(); } while (ids.Count > 0); if (end) { break; } } } return(new BulkOperationResult { Total = progress.Processed }); }
protected async Task <IOperationResult> ExecuteOperation(string collectionName, CollectionOperationOptions options, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, Func <LazyStringValue, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token) { const int batchSize = 1024; var progress = new DeterminateProgress(); var cancellationToken = token.Token; var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection; long lastEtag; long totalCount; using (context.OpenReadTransaction()) { lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs); totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs); } progress.Total = totalCount; // send initial progress with total count set, and 0 as processed count onProgress(progress); long startEtag = 0; using (var rateGate = options.MaxOpsPerSecond.HasValue ? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1)) : null) { var end = false; while (startEtag <= lastEtag) { cancellationToken.ThrowIfCancellationRequested(); using (context.OpenReadTransaction()) { var ids = new Queue <LazyStringValue>(batchSize); foreach (var document in GetDocuments(context, collectionName, startEtag, batchSize, isAllDocs)) { cancellationToken.ThrowIfCancellationRequested(); token.Delay(); if (isAllDocs && IsHiLoDocument(document)) { continue; } if (document.Etag > lastEtag) // we don't want to go over the documents that we have patched { end = true; break; } startEtag = document.Etag + 1; ids.Enqueue(document.Id); } if (ids.Count == 0) { break; } do { var command = new ExecuteRateLimitedOperations <LazyStringValue>(ids, action, rateGate, token); await Database.TxMerger.Enqueue(command); progress.Processed += command.Processed; onProgress(progress); if (command.NeedWait) { rateGate?.WaitToProceed(); } } while (ids.Count > 0); if (end) { break; } } } } return(new BulkOperationResult { Total = progress.Processed }); }