public RavenJArray DeleteByIndex(string indexName, IndexQuery queryToDelete, BulkOperationOptions options = null) { return PerformBulkOperation(indexName, queryToDelete, options, (docId, tx) => { database.Documents.Delete(docId, null, tx); return new { Document = docId, Deleted = true }; }); }
public RavenJArray UpdateByIndex(string indexName, IndexQuery queryToUpdate, ScriptedPatchRequest patch, BulkOperationOptions options = null) { return PerformBulkOperation(indexName, queryToUpdate, options, (docId, tx) => { var patchResult = database.Patches.ApplyPatch(docId, null, patch, tx); return new { Document = docId, Result = patchResult.Item1, Debug = patchResult.Item2 }; }); }
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, BulkOperationOptions options, Func<string, TransactionInformation, object> batchOperation) { options = options ?? new BulkOperationOptions(); var array = new RavenJArray(); var bulkIndexQuery = new IndexQuery { Query = indexQuery.Query, Start = indexQuery.Start, Cutoff = indexQuery.Cutoff ?? SystemTime.UtcNow, WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow, PageSize = int.MaxValue, FieldsToFetch = new[] { Constants.DocumentIdFieldName }, SortedFields = indexQuery.SortedFields, HighlighterPreTags = indexQuery.HighlighterPreTags, HighlighterPostTags = indexQuery.HighlighterPostTags, HighlightedFields = indexQuery.HighlightedFields, SortHints = indexQuery.SortHints }; bool stale; var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale); if (stale && options.AllowStale == false) { if (options.StaleTimeout != null) { var staleWaitTimeout = Stopwatch.StartNew(); while (stale && staleWaitTimeout.Elapsed < options.StaleTimeout) { queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale); if(stale) SystemTime.Wait(100); } } if (stale) { if (options.StaleTimeout != null) throw new InvalidOperationException("Bulk operation cancelled because the index is stale and StaleTimout of " + options.StaleTimeout + "passed"); throw new InvalidOperationException("Bulk operation cancelled because the index is stale and allowStale is false"); } } var token = tokenSource.Token; const int batchSize = 1024; int maxOpsPerSec = options.MaxOpsPerSec ?? int.MaxValue; using (var enumerator = queryResults.GetEnumerator()) { var duration = Stopwatch.StartNew(); var operations = 0; while (true) { database.WorkContext.UpdateFoundWork(); if (timeout != null) timeout.Delay(); var batchCount = 0; var shouldWaitNow = false; token.ThrowIfCancellationRequested(); using (database.DocumentLock.Lock()) { database.TransactionalStorage.Batch(actions => { while (batchCount < batchSize && enumerator.MoveNext()) { batchCount++; operations++; var result = batchOperation(enumerator.Current, transactionInformation); if(options.RetrieveDetails) array.Add(RavenJObject.FromObject(result)); if (operations >= maxOpsPerSec && duration.ElapsedMilliseconds < 1000) { shouldWaitNow = true; break; } } }); if (shouldWaitNow) { SystemTime.Wait(500); operations = 0; duration.Restart(); continue; } } if (batchCount < batchSize) break; } } return array; }
private HttpResponseMessage OnBulkOperation(Func<string, IndexQuery, BulkOperationOptions, RavenJArray> batchOperation, string index, CancellationTimeout timeout) { if (string.IsNullOrEmpty(index)) return GetEmptyMessage(HttpStatusCode.BadRequest); var option = new BulkOperationOptions { AllowStale = GetAllowStale(), MaxOpsPerSec = GetMaxOpsPerSec(), StaleTimeout = GetStaleTimeout(), RetrieveDetails = GetRetrieveDetails() }; var indexQuery = GetIndexQuery(maxPageSize: int.MaxValue); var status = new BulkOperationStatus(); long id; var task = Task.Factory.StartNew(() => { status.State = batchOperation(index, indexQuery, option); }).ContinueWith(t => { if (timeout != null) timeout.Dispose(); if (t.IsFaulted == false) { status.Completed = true; return; } var exception = t.Exception.ExtractSingleInnerException(); status.State = RavenJObject.FromObject(new { Error = exception.Message }); status.Faulted = true; status.Completed = true; }); Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.IndexBulkOperation, Payload = index }, out id, timeout.CancellationTokenSource); return GetMessageWithObject(new { OperationId = id }); }