public RavenJArray DeleteByIndex(string indexName, IndexQuery queryToDelete, BulkOperationOptions options = null)
{
return PerformBulkOperation(indexName, queryToDelete, options, (docId, tx) =>
{
database.Documents.Delete(docId, null, tx);
return new { Document = docId, Deleted = true };
});
}
public RavenJArray UpdateByIndex(string indexName, IndexQuery queryToUpdate, ScriptedPatchRequest patch, BulkOperationOptions options = null)
{
return PerformBulkOperation(indexName, queryToUpdate, options, (docId, tx) =>
{
var patchResult = database.Patches.ApplyPatch(docId, null, patch, tx);
return new { Document = docId, Result = patchResult.Item1, Debug = patchResult.Item2 };
});
}
private RavenJArray PerformBulkOperation(string index, IndexQuery indexQuery, BulkOperationOptions options, Func<string, TransactionInformation, object> batchOperation)
{
options = options ?? new BulkOperationOptions();
var array = new RavenJArray();
var bulkIndexQuery = new IndexQuery
{
Query = indexQuery.Query,
Start = indexQuery.Start,
Cutoff = indexQuery.Cutoff ?? SystemTime.UtcNow,
WaitForNonStaleResultsAsOfNow = indexQuery.WaitForNonStaleResultsAsOfNow,
PageSize = int.MaxValue,
FieldsToFetch = new[] { Constants.DocumentIdFieldName },
SortedFields = indexQuery.SortedFields,
HighlighterPreTags = indexQuery.HighlighterPreTags,
HighlighterPostTags = indexQuery.HighlighterPostTags,
HighlightedFields = indexQuery.HighlightedFields,
SortHints = indexQuery.SortHints
};
bool stale;
var queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
if (stale && options.AllowStale == false)
{
if (options.StaleTimeout != null)
{
var staleWaitTimeout = Stopwatch.StartNew();
while (stale && staleWaitTimeout.Elapsed < options.StaleTimeout)
{
queryResults = database.Queries.QueryDocumentIds(index, bulkIndexQuery, tokenSource, out stale);
if(stale)
SystemTime.Wait(100);
}
}
if (stale)
{
if (options.StaleTimeout != null)
throw new InvalidOperationException("Bulk operation cancelled because the index is stale and StaleTimout of " + options.StaleTimeout + "passed");
throw new InvalidOperationException("Bulk operation cancelled because the index is stale and allowStale is false");
}
}
var token = tokenSource.Token;
const int batchSize = 1024;
int maxOpsPerSec = options.MaxOpsPerSec ?? int.MaxValue;
using (var enumerator = queryResults.GetEnumerator())
{
var duration = Stopwatch.StartNew();
var operations = 0;
while (true)
{
database.WorkContext.UpdateFoundWork();
if (timeout != null)
timeout.Delay();
var batchCount = 0;
var shouldWaitNow = false;
token.ThrowIfCancellationRequested();
using (database.DocumentLock.Lock())
{
database.TransactionalStorage.Batch(actions =>
{
while (batchCount < batchSize && enumerator.MoveNext())
{
batchCount++;
operations++;
var result = batchOperation(enumerator.Current, transactionInformation);
if(options.RetrieveDetails)
array.Add(RavenJObject.FromObject(result));
if (operations >= maxOpsPerSec && duration.ElapsedMilliseconds < 1000)
{
shouldWaitNow = true;
break;
}
}
});
if (shouldWaitNow)
{
SystemTime.Wait(500);
operations = 0;
duration.Restart();
continue;
}
}
if (batchCount < batchSize) break;
}
}
return array;
}
private HttpResponseMessage OnBulkOperation(Func<string, IndexQuery, BulkOperationOptions, RavenJArray> batchOperation, string index, CancellationTimeout timeout)
{
if (string.IsNullOrEmpty(index))
return GetEmptyMessage(HttpStatusCode.BadRequest);
var option = new BulkOperationOptions
{
AllowStale = GetAllowStale(),
MaxOpsPerSec = GetMaxOpsPerSec(),
StaleTimeout = GetStaleTimeout(),
RetrieveDetails = GetRetrieveDetails()
};
var indexQuery = GetIndexQuery(maxPageSize: int.MaxValue);
var status = new BulkOperationStatus();
long id;
var task = Task.Factory.StartNew(() =>
{
status.State = batchOperation(index, indexQuery, option);
}).ContinueWith(t =>
{
if (timeout != null)
timeout.Dispose();
if (t.IsFaulted == false)
{
status.Completed = true;
return;
}
var exception = t.Exception.ExtractSingleInnerException();
status.State = RavenJObject.FromObject(new { Error = exception.Message });
status.Faulted = true;
status.Completed = true;
});
Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription
{
StartTime = SystemTime.UtcNow,
TaskType = TaskActions.PendingTaskType.IndexBulkOperation,
Payload = index
}, out id, timeout.CancellationTokenSource);
return GetMessageWithObject(new { OperationId = id });
}