public DatabaseBulkOperations(DocumentDatabase database, TransactionInformation transactionInformation, CancellationTokenSource tokenSource, CancellationTimeout timeout) { this.database = database; this.transactionInformation = transactionInformation; this.tokenSource = tokenSource; this.timeout = timeout; }
public StreamQueryContent(HttpRequestMessage req, QueryActions.DatabaseQueryOperation queryOp, IStorageActionsAccessor accessor, CancellationTimeout timeout, Action <string> contentTypeSetter) { this.req = req; this.queryOp = queryOp; this.accessor = accessor; _timeout = timeout; outputContentTypeSetter = contentTypeSetter; }
public DatabaseBulkOperations(DocumentDatabase database, CancellationTokenSource tokenSource, CancellationTimeout timeout) { this.database = database; this.tokenSource = tokenSource; this.timeout = timeout; }
public int BulkInsert(BulkInsertOptions options, IEnumerable <IEnumerable <JsonDocument> > docBatches, Guid operationId, CancellationToken token, CancellationTimeout timeout = null) { var documents = 0; Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification { OperationId = operationId, Type = DocumentChangeTypes.BulkInsertStarted }); using (var cts = CancellationTokenSource.CreateLinkedTokenSource(token, WorkContext.CancellationToken)) { foreach (var docs in docBatches) { cts.Token.ThrowIfCancellationRequested(); var docsToInsert = docs.ToArray(); var batch = 0; var keys = new HashSet <string>(StringComparer.OrdinalIgnoreCase); var collectionsAndEtags = new Dictionary <string, Etag>(StringComparer.OrdinalIgnoreCase); timeout?.Pause(); using (Database.DocumentLock.Lock()) { timeout?.Resume(); TransactionalStorage.Batch(accessor => { var inserts = 0; foreach (var doc in docsToInsert) { try { if (string.IsNullOrWhiteSpace(doc.Key)) { throw new InvalidOperationException("Cannot try to bulk insert a document without a key"); } doc.Key = doc.Key.Trim(); if (doc.Key[doc.Key.Length - 1] == '/') { doc.Key += GetNextIdentityValueWithoutOverwritingOnExistingDocuments(doc.Key, accessor); } RemoveReservedProperties(doc.DataAsJson); RemoveMetadataReservedProperties(doc.Metadata); if (options.CheckReferencesInIndexes) { keys.Add(doc.Key); } documents++; batch++; AssertPutOperationNotVetoed(doc.Key, doc.Metadata, doc.DataAsJson, null); if (options.OverwriteExisting && options.SkipOverwriteIfUnchanged) { var existingDoc = accessor.Documents.DocumentByKey(doc.Key); if (IsTheSameDocument(doc, existingDoc)) { continue; } } foreach (var trigger in Database.PutTriggers) { trigger.Value.OnPut(doc.Key, doc.DataAsJson, doc.Metadata, null); } var result = accessor.Documents.InsertDocument(doc.Key, doc.DataAsJson, doc.Metadata, options.OverwriteExisting); if (result.Updated == false) { inserts++; } doc.Etag = result.Etag; doc.Metadata.EnsureSnapshot( "Metadata was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?"); doc.DataAsJson.EnsureSnapshot( "Document was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?"); var entityName = doc.Metadata.Value <string>(Constants.RavenEntityName); Etag highestEtagInCollection; if (string.IsNullOrEmpty(entityName) == false && (collectionsAndEtags.TryGetValue(entityName, out highestEtagInCollection) == false || result.Etag.CompareTo(highestEtagInCollection) > 0)) { collectionsAndEtags[entityName] = result.Etag; } foreach (var trigger in Database.PutTriggers) { trigger.Value.AfterPut(doc.Key, doc.DataAsJson, doc.Metadata, result.Etag, null); } Database.WorkContext.UpdateFoundWork(); } catch (Exception e) { Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification { OperationId = operationId, Message = e.Message, Etag = doc.Etag, Id = doc.Key, Type = DocumentChangeTypes.BulkInsertError }); throw; } } if (options.CheckReferencesInIndexes) { foreach (var key in keys) { Database.Indexes.CheckReferenceBecauseOfDocumentUpdate(key, accessor); } } accessor.Documents.IncrementDocumentCount(inserts); }); foreach (var collectionEtagPair in collectionsAndEtags) { Database.LastCollectionEtags.Update(collectionEtagPair.Key, collectionEtagPair.Value); } WorkContext.ShouldNotifyAboutWork(() => "BulkInsert batch of " + batch + " docs"); WorkContext.NotifyAboutWork(); // forcing notification so we would start indexing right away WorkContext.UpdateFoundWork(); } } } Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification { OperationId = operationId, Type = DocumentChangeTypes.BulkInsertEnded }); if (documents > 0) { WorkContext.ShouldNotifyAboutWork(() => "BulkInsert of " + documents + " docs"); } return(documents); }
private IEnumerable <TimeSeriesAppend> YieldBatchItems(Stream partialStream, JsonSerializer serializer, CancellationTimeout timeout, Action <int> changeTimeSeriesFunc) { using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true)) { var reader = new BinaryReader(stream); var count = reader.ReadInt32(); for (var i = 0; i < count; i++) { timeout.Delay(); var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader) { DateTimeKindHandling = DateTimeKind.Unspecified }); yield return(doc.ToObject <TimeSeriesAppend>(serializer)); } changeTimeSeriesFunc(count); } }
private IEnumerable <IEnumerable <TimeSeriesAppend> > YieldChangeBatches(Stream requestStream, CancellationTimeout timeout, Action <int> changeTimeSeriesFunc) { var serializer = JsonExtensions.CreateDefaultJsonSerializer(); try { using (requestStream) { var binaryReader = new BinaryReader(requestStream); while (true) { timeout.ThrowIfCancellationRequested(); int batchSize; try { batchSize = binaryReader.ReadInt32(); } catch (EndOfStreamException) { break; } using (var stream = new PartialStream(requestStream, batchSize)) { yield return(YieldBatchItems(stream, serializer, timeout, changeTimeSeriesFunc)); } } } } finally { requestStream.Close(); } }
private HttpResponseMessage OnBulkOperation(Func <string, IndexQuery, BulkOperationOptions, Action <BulkOperationProgress>, RavenJArray> batchOperation, string index, CancellationTimeout timeout) { if (string.IsNullOrEmpty(index)) { return(GetEmptyMessage(HttpStatusCode.BadRequest)); } var option = new BulkOperationOptions { AllowStale = GetAllowStale(), MaxOpsPerSec = GetMaxOpsPerSec(), StaleTimeout = GetStaleTimeout(), RetrieveDetails = GetRetrieveDetails() }; var indexQuery = GetIndexQuery(maxPageSize: int.MaxValue); var status = new BulkOperationStatus(); long id; var task = Task.Factory.StartNew(() => { using (DocumentCacher.SkipSetDocumentsInDocumentCache()) { status.State["Batch"] = batchOperation(index, indexQuery, option, x => { status.MarkProgress(x); }); } }).ContinueWith(t => { if (timeout != null) { timeout.Dispose(); } if (t.IsFaulted == false) { status.MarkCompleted($"Processed {status.OperationProgress.ProcessedEntries} items"); return; } var exception = t.Exception.ExtractSingleInnerException(); status.MarkFaulted(exception.Message); }); Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.IndexBulkOperation, Description = index }, out id, timeout.CancellationTokenSource); return(GetMessageWithObject(new { OperationId = id }, HttpStatusCode.Accepted)); }
private HttpResponseMessage OnBulkOperation(Func <string, IndexQuery, BulkOperationOptions, RavenJArray> batchOperation, string index, CancellationTimeout timeout) { if (string.IsNullOrEmpty(index)) { return(GetEmptyMessage(HttpStatusCode.BadRequest)); } var option = new BulkOperationOptions { AllowStale = GetAllowStale(), MaxOpsPerSec = GetMaxOpsPerSec(), StaleTimeout = GetStaleTimeout(), RetrieveDetails = GetRetrieveDetails() }; var indexQuery = GetIndexQuery(maxPageSize: int.MaxValue); var status = new BulkOperationStatus(); long id; var task = Task.Factory.StartNew(() => { status.State = batchOperation(index, indexQuery, option); }).ContinueWith(t => { if (timeout != null) { timeout.Dispose(); } if (t.IsFaulted == false) { status.Completed = true; return; } var exception = t.Exception.ExtractSingleInnerException(); status.State = RavenJObject.FromObject(new { Error = exception.Message }); status.Faulted = true; status.Completed = true; }); Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.IndexBulkOperation, Payload = index }, out id, timeout.CancellationTokenSource); return(GetMessageWithObject(new { OperationId = id })); }