public Task <IOperationResult> ExecutePatch(string collectionName, CollectionOperationOptions options, PatchRequest patch,
BlittableJsonReaderObject patchArgs, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
return(ExecuteOperation(collectionName, options, Context, onProgress,
key => new PatchDocumentCommand(Context, key, null, false, (patch, patchArgs), (null, null),
Database, false, false), token));
}
public override Task <IOperationResult> ExecuteDelete(string collectionName, CollectionOperationOptions options, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
if (_excludeIds.Count == 0)
{
return(base.ExecuteDelete(collectionName, options, onProgress, token));
}
// specific collection w/ exclusions
return(ExecuteOperation(collectionName, options, Context, onProgress, key =>
{
if (_excludeIds.Contains(key) == false)
{
return new DeleteDocumentCommand(key, null, Database);
}
return null;
}, token));
}
private void ExecuteCollectionOperation(Func <CollectionRunner, string, CollectionOperationOptions, Action <IOperationProgress>, OperationCancelToken, Task <IOperationResult> > operation, DocumentsOperationContext context, IDisposable returnContextToPool, Documents.Operations.Operations.OperationType operationType, HashSet <string> excludeIds)
{
var collectionName = GetStringQueryString("name");
var token = CreateTimeLimitedCollectionOperationToken();
var collectionRunner = new StudioCollectionRunner(Database, context, excludeIds);
var operationId = Database.Operations.GetNextOperationId();
// use default options
var options = new CollectionOperationOptions();
var task = Database.Operations.AddOperation(Database, collectionName, operationType, onProgress =>
operation(collectionRunner, collectionName, options, onProgress, token), operationId, token: token);
task.ContinueWith(_ => returnContextToPool.Dispose());
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteOperationId(context, operationId);
}
}
protected async Task <IOperationResult> ExecuteOperation(string collectionName, long start, long take, CollectionOperationOptions options, DocumentsOperationContext context,
Action <DeterminateProgress> onProgress, Func <string, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token)
{
var progress = new DeterminateProgress();
var cancellationToken = token.Token;
var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection;
long lastEtag;
long totalCount;
using (context.OpenReadTransaction())
{
lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs);
totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs);
}
progress.Total = totalCount;
// send initial progress with total count set, and 0 as processed count
onProgress(progress);
long startEtag = 0;
var alreadySeenIdsCount = new Reference <long>();
string startAfterId = null;
using (var rateGate = options.MaxOpsPerSecond.HasValue
? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1))
: null)
{
var end = false;
var ids = new Queue <string>(OperationBatchSize);
while (startEtag <= lastEtag)
{
cancellationToken.ThrowIfCancellationRequested();
ids.Clear();
Database.ForTestingPurposes?.CollectionRunnerBeforeOpenReadTransaction?.Invoke();
using (context.OpenReadTransaction())
{
foreach (var document in GetDocuments(context, collectionName, startEtag, startAfterId, alreadySeenIdsCount, OperationBatchSize, isAllDocs, DocumentFields.Id, out bool isStartsWithOrIdQuery))
{
using (document)
{
cancellationToken.ThrowIfCancellationRequested();
token.Delay();
if (isAllDocs && document.Id.StartsWith(HiLoHandler.RavenHiloIdPrefix, StringComparison.OrdinalIgnoreCase))
{
continue;
}
// start with and id queries aren't ordered by the etag
if (isStartsWithOrIdQuery == false && document.Etag > lastEtag)
{
// we don't want to go over the documents that we have patched
end = true;
break;
}
startEtag = document.Etag + 1;
if (start > 0)
{
start--;
continue;
}
if (take-- <= 0)
{
end = true;
break;
}
startAfterId = document.Id;
ids.Enqueue(document.Id);
context.Transaction.ForgetAbout(document);
}
}
}
if (ids.Count == 0)
{
break;
}
do
{
var command = new ExecuteRateLimitedOperations <string>(ids, action, rateGate, token,
maxTransactionSize: 16 * Voron.Global.Constants.Size.Megabyte,
batchSize: OperationBatchSize);
await Database.TxMerger.Enqueue(command);
progress.Processed += command.Processed;
onProgress(progress);
if (command.NeedWait)
{
rateGate?.WaitToProceed();
}
token.Delay();
} while (ids.Count > 0);
if (end)
{
break;
}
}
}
return(new BulkOperationResult
{
Total = progress.Processed
});
}
public Task <IOperationResult> ExecutePatch(string collectionName, long start, long take, CollectionOperationOptions options, PatchRequest patch,
BlittableJsonReaderObject patchArgs, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
return(ExecuteOperation(collectionName, start, take, options, Context, onProgress,
key => new PatchDocumentCommand(Context, key, expectedChangeVector: null, skipPatchIfChangeVectorMismatch: false, patch: (patch, patchArgs), patchIfMissing: (null, null), createIfMissing: null, Database, isTest: false, debugMode: false, collectResultsNeeded: false, returnDocument: false), token));
}
public virtual Task <IOperationResult> ExecuteDelete(string collectionName, long start, long take, CollectionOperationOptions options, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
return(ExecuteOperation(collectionName, start, take, options, Context, onProgress, key => new DeleteDocumentCommand(key, null, Database), token));
}
protected async Task <IOperationResult> ExecuteOperation(string collectionName, CollectionOperationOptions options, DocumentsOperationContext context,
Action <DeterminateProgress> onProgress, Func <LazyStringValue, TransactionOperationsMerger.MergedTransactionCommand> action, OperationCancelToken token)
{
const int batchSize = 1024;
var progress = new DeterminateProgress();
var cancellationToken = token.Token;
var isAllDocs = collectionName == Constants.Documents.Collections.AllDocumentsCollection;
long lastEtag;
long totalCount;
using (context.OpenReadTransaction())
{
lastEtag = GetLastEtagForCollection(context, collectionName, isAllDocs);
totalCount = GetTotalCountForCollection(context, collectionName, isAllDocs);
}
progress.Total = totalCount;
// send initial progress with total count set, and 0 as processed count
onProgress(progress);
long startEtag = 0;
using (var rateGate = options.MaxOpsPerSecond.HasValue
? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1))
: null)
{
var end = false;
while (startEtag <= lastEtag)
{
cancellationToken.ThrowIfCancellationRequested();
using (context.OpenReadTransaction())
{
var ids = new Queue <LazyStringValue>(batchSize);
foreach (var document in GetDocuments(context, collectionName, startEtag, batchSize, isAllDocs))
{
cancellationToken.ThrowIfCancellationRequested();
token.Delay();
if (isAllDocs && IsHiLoDocument(document))
{
continue;
}
if (document.Etag > lastEtag) // we don't want to go over the documents that we have patched
{
end = true;
break;
}
startEtag = document.Etag + 1;
ids.Enqueue(document.Id);
}
if (ids.Count == 0)
{
break;
}
do
{
var command = new ExecuteRateLimitedOperations <LazyStringValue>(ids, action, rateGate, token);
await Database.TxMerger.Enqueue(command);
progress.Processed += command.Processed;
onProgress(progress);
if (command.NeedWait)
{
rateGate?.WaitToProceed();
}
} while (ids.Count > 0);
if (end)
{
break;
}
}
}
}
return(new BulkOperationResult
{
Total = progress.Processed
});
}
private IOperationResult ExecuteOperation(string collectionName, CollectionOperationOptions options, DocumentsOperationContext context,
Action <DeterminateProgress> onProgress, Action <string> action, OperationCancelToken token)
{
const int batchSize = 1024;
var progress = new DeterminateProgress();
var cancellationToken = token.Token;
long lastEtag;
long totalCount;
using (context.OpenReadTransaction())
{
lastEtag = _database.DocumentsStorage.GetLastDocumentEtag(context, collectionName);
_database.DocumentsStorage.GetNumberOfDocumentsToProcess(context, collectionName, 0, out totalCount);
}
progress.Total = totalCount;
long startEtag = 0;
using (var rateGate = options.MaxOpsPerSecond.HasValue
? new RateGate(options.MaxOpsPerSecond.Value, TimeSpan.FromSeconds(1))
: null)
{
bool done = false;
//The reason i do this nested loop is because i can't operate on a document while iterating the document tree.
while (startEtag <= lastEtag)
{
cancellationToken.ThrowIfCancellationRequested();
bool wait = false;
using (var tx = context.OpenWriteTransaction())
{
var documents = _database.DocumentsStorage.GetDocumentsFrom(context, collectionName, startEtag, 0, batchSize).ToList();
foreach (var document in documents)
{
cancellationToken.ThrowIfCancellationRequested();
if (document.Etag > lastEtag)// we don't want to go over the documents that we have patched
{
done = true;
break;
}
if (rateGate != null && rateGate.WaitToProceed(0) == false)
{
wait = true;
break;
}
startEtag = document.Etag;
action(document.Key);
progress.Processed++;
}
tx.Commit();
onProgress(progress);
if (wait)
{
rateGate.WaitToProceed();
}
if (done || documents.Count == 0)
{
break;
}
}
}
}
return(new BulkOperationResult
{
Total = progress.Processed
});
}
public IOperationResult ExecutePatch(string collectionName, CollectionOperationOptions options, PatchRequest patch, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
return(ExecuteOperation(collectionName, options, _context, onProgress, key => _database.Patch.Apply(context, key, null, patch, null), token));
}
public IOperationResult ExecuteDelete(string collectionName, CollectionOperationOptions options, DocumentsOperationContext documentsOperationContext, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
return(ExecuteOperation(collectionName, options, _context, onProgress, key => _database.DocumentsStorage.Delete(_context, key, null), token));
}
