public Task <IOperationResult> ExecutePatch(string collectionName, CollectionOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, Action <IOperationProgress> onProgress, OperationCancelToken token) { return(ExecuteOperation(collectionName, options, Context, onProgress, key => new PatchDocumentCommand(Context, key, null, false, (patch, patchArgs), (null, null), Database, false, false), token)); }
public override Task <IOperationResult> ExecuteDelete(string collectionName, CollectionOperationOptions options, Action <IOperationProgress> onProgress, OperationCancelToken token) { if (_excludeIds.Count == 0) { return(base.ExecuteDelete(collectionName, options, onProgress, token)); } // specific collection w/ exclusions return(ExecuteOperation(collectionName, options, Context, onProgress, key => { if (_excludeIds.Contains(key) == false) { return new DeleteDocumentCommand(key, null, Database); } return null; }, token)); }
private void ExecuteCollectionOperation(Func <CollectionRunner, string, CollectionOperationOptions, Action <IOperationProgress>, OperationCancelToken, Task <IOperationResult> > operation, DocumentsOperationContext context, IDisposable returnContextToPool, Documents.Operations.Operations.OperationType operationType, HashSet <string> excludeIds) { var collectionName = GetStringQueryString("name"); var token = CreateTimeLimitedCollectionOperationToken(); var collectionRunner = new StudioCollectionRunner(Database, context, excludeIds); var operationId = Database.Operations.GetNextOperationId(); // use default options var options = new CollectionOperationOptions(); var task = Database.Operations.AddOperation(Database, collectionName, operationType, onProgress => operation(collectionRunner, collectionName, options, onProgress, token), operationId, token: token); task.ContinueWith(_ => returnContextToPool.Dispose()); using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteOperationId(context, operationId); } }
protected async Task <IOperationResult> ExecuteOperation(string collectionName, long start, long take, CollectionOperationOptions options, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, Func <string, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token) { var progress = new DeterminateProgress(); var cancellationToken = token.Token; var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection; long lastEtag; long totalCount; using (context.OpenReadTransaction()) { lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs); totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs); } progress.Total = totalCount; // send initial progress with total count set, and 0 as processed count onProgress(progress); long startEtag = 0; var alreadySeenIdsCount = new Reference <long>(); string startAfterId = null; using (var rateGate = options.MaxOpsPerSecond.HasValue ? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1)) : null) { var end = false; var ids = new Queue <string>(OperationBatchSize); while (startEtag <= lastEtag) { cancellationToken.ThrowIfCancellationRequested(); ids.Clear(); Database.ForTestingPurposes?.CollectionRunnerBeforeOpenReadTransaction?.Invoke(); using (context.OpenReadTransaction()) { foreach (var document in GetDocuments(context, collectionName, startEtag, startAfterId, alreadySeenIdsCount, OperationBatchSize, isAllDocs, DocumentFields.Id, out bool isStartsWithOrIdQuery)) { using (document) { cancellationToken.ThrowIfCancellationRequested(); token.Delay(); if (isAllDocs && document.Id.StartsWith(HiLoHandler.RavenHiloIdPrefix, StringComparison.OrdinalIgnoreCase)) { continue; } // start with and id queries aren't ordered by the etag if (isStartsWithOrIdQuery == false && document.Etag > lastEtag) { // we don't want to go over the documents that we have patched end = true; break; } startEtag = document.Etag + 1; if (start > 0) { start--; continue; } if (take-- <= 0) { end = true; break; } startAfterId = document.Id; ids.Enqueue(document.Id); context.Transaction.ForgetAbout(document); } } } if (ids.Count == 0) { break; } do { var command = new ExecuteRateLimitedOperations <string>(ids, action, rateGate, token, maxTransactionSize: 16 * Voron.Global.Constants.Size.Megabyte, batchSize: OperationBatchSize); await Database.TxMerger.Enqueue(command); progress.Processed += command.Processed; onProgress(progress); if (command.NeedWait) { rateGate?.WaitToProceed(); } token.Delay(); } while (ids.Count > 0); if (end) { break; } } } return(new BulkOperationResult { Total = progress.Processed }); }
public Task <IOperationResult> ExecutePatch(string collectionName, long start, long take, CollectionOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, Action <IOperationProgress> onProgress, OperationCancelToken token) { return(ExecuteOperation(collectionName, start, take, options, Context, onProgress, key => new PatchDocumentCommand(Context, key, expectedChangeVector: null, skipPatchIfChangeVectorMismatch: false, patch: (patch, patchArgs), patchIfMissing: (null, null), createIfMissing: null, Database, isTest: false, debugMode: false, collectResultsNeeded: false, returnDocument: false), token)); }
public virtual Task <IOperationResult> ExecuteDelete(string collectionName, long start, long take, CollectionOperationOptions options, Action <IOperationProgress> onProgress, OperationCancelToken token) { return(ExecuteOperation(collectionName, start, take, options, Context, onProgress, key => new DeleteDocumentCommand(key, null, Database), token)); }
protected async Task <IOperationResult> ExecuteOperation(string collectionName, CollectionOperationOptions options, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, Func <LazyStringValue, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token) { const int batchSize = 1024; var progress = new DeterminateProgress(); var cancellationToken = token.Token; var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection; long lastEtag; long totalCount; using (context.OpenReadTransaction()) { lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs); totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs); } progress.Total = totalCount; // send initial progress with total count set, and 0 as processed count onProgress(progress); long startEtag = 0; using (var rateGate = options.MaxOpsPerSecond.HasValue ? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1)) : null) { var end = false; while (startEtag <= lastEtag) { cancellationToken.ThrowIfCancellationRequested(); using (context.OpenReadTransaction()) { var ids = new Queue <LazyStringValue>(batchSize); foreach (var document in GetDocuments(context, collectionName, startEtag, batchSize, isAllDocs)) { cancellationToken.ThrowIfCancellationRequested(); token.Delay(); if (isAllDocs && IsHiLoDocument(document)) { continue; } if (document.Etag > lastEtag) // we don't want to go over the documents that we have patched { end = true; break; } startEtag = document.Etag + 1; ids.Enqueue(document.Id); } if (ids.Count == 0) { break; } do { var command = new ExecuteRateLimitedOperations <LazyStringValue>(ids, action, rateGate, token); await Database.TxMerger.Enqueue(command); progress.Processed += command.Processed; onProgress(progress); if (command.NeedWait) { rateGate?.WaitToProceed(); } } while (ids.Count > 0); if (end) { break; } } } } return(new BulkOperationResult { Total = progress.Processed }); }
private IOperationResult ExecuteOperation(string collectionName, CollectionOperationOptions options, DocumentsOperationContext context, Action <DeterminateProgress> onProgress, Action <string> action, OperationCancelToken token) { const int batchSize = 1024; var progress = new DeterminateProgress(); var cancellationToken = token.Token; long lastEtag; long totalCount; using (context.OpenReadTransaction()) { lastEtag = _database.DocumentsStorage.GetLastDocumentEtag(context, collectionName); _database.DocumentsStorage.GetNumberOfDocumentsToProcess(context, collectionName, 0, out totalCount); } progress.Total = totalCount; long startEtag = 0; using (var rateGate = options.MaxOpsPerSecond.HasValue ? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1)) : null) { bool done = false; //The reason i do this nested loop is because i can't operate on a document while iterating the document tree. while (startEtag <= lastEtag) { cancellationToken.ThrowIfCancellationRequested(); bool wait = false; using (var tx = context.OpenWriteTransaction()) { var documents = _database.DocumentsStorage.GetDocumentsFrom(context, collectionName, startEtag, 0, batchSize).ToList(); foreach (var document in documents) { cancellationToken.ThrowIfCancellationRequested(); if (document.Etag > lastEtag)// we don't want to go over the documents that we have patched { done = true; break; } if (rateGate != null && rateGate.WaitToProceed(0) == false) { wait = true; break; } startEtag = document.Etag; action(document.Key); progress.Processed++; } tx.Commit(); onProgress(progress); if (wait) { rateGate.WaitToProceed(); } if (done || documents.Count == 0) { break; } } } } return(new BulkOperationResult { Total = progress.Processed }); }
public IOperationResult ExecutePatch(string collectionName, CollectionOperationOptions options, PatchRequest patch, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token) { return(ExecuteOperation(collectionName, options, _context, onProgress, key => _database.Patch.Apply(context, key, null, patch, null), token)); }
public IOperationResult ExecuteDelete(string collectionName, CollectionOperationOptions options, DocumentsOperationContext documentsOperationContext, Action <IOperationProgress> onProgress, OperationCancelToken token) { return(ExecuteOperation(collectionName, options, _context, onProgress, key => _database.DocumentsStorage.Delete(_context, key, null), token)); }