public DatabaseBulkOperations(DocumentDatabase database, TransactionInformation transactionInformation, CancellationTokenSource tokenSource, CancellationTimeout timeout) { this.database = database; this.transactionInformation = transactionInformation; this.tokenSource = tokenSource; this.timeout = timeout; }
private static void WriteToStream(JsonWriter writer, RavenJObject item, CancellationTimeout timeout) { timeout.Delay(); item.WriteTo(writer); }
public StreamQueryContent(HttpRequestMessage req, QueryActions.DatabaseQueryOperation queryOp, IStorageActionsAccessor accessor, CancellationTimeout timeout, Action<string> contentTypeSetter) { this.req = req; this.queryOp = queryOp; this.accessor = accessor; _timeout = timeout; outputContentTypeSetter = contentTypeSetter; }
private IEnumerable<JsonDocument> YieldDocumentsInBatch(CancellationTimeout timeout, Stream partialStream, Action<int> increaseDocumentsCount) { using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true)) { var reader = new BinaryReader(stream); var count = reader.ReadInt32(); for (var i = 0; i < count; i++) { timeout.Delay(); var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader) { DateTimeKindHandling = DateTimeKind.Unspecified }); var metadata = doc.Value<RavenJObject>("@metadata"); if (metadata == null) throw new InvalidOperationException("Could not find metadata for document"); var id = metadata.Value<string>("@id"); if (string.IsNullOrEmpty(id)) throw new InvalidOperationException("Could not get id from metadata"); if (id.Equals(Constants.BulkImportHeartbeatDocKey, StringComparison.InvariantCultureIgnoreCase)) continue; //its just a token document, should not get written into the database //the purpose of the heartbeat document is to make sure that the connection doesn't time-out //during long pauses in the bulk insert operation. // Currently used by smuggler to make sure that the connection doesn't time out if there is a //continuation token and lots of document skips doc.Remove("@metadata"); yield return new JsonDocument { Key = id, DataAsJson = doc, Metadata = metadata }; } increaseDocumentsCount(count); } }
private IEnumerable<IEnumerable<JsonDocument>> YieldBatches(CancellationTimeout timeout, Stream inputStream, ManualResetEventSlim mre, Action<int> increaseDocumentsCount) { try { using (inputStream) { var binaryReader = new BinaryReader(inputStream); while (true) { timeout.ThrowIfCancellationRequested(); int size; try { size = binaryReader.ReadInt32(); } catch (EndOfStreamException) { break; } using (var stream = new PartialStream(inputStream, size)) { yield return YieldDocumentsInBatch(timeout, stream, increaseDocumentsCount); } } } } finally { mre.Set(); inputStream.Close(); } }
private IEnumerable<JsonDocument> YieldDocumentsInBatch(CancellationTimeout timeout, Stream partialStream, Action<int> increaseDocumentsCount) { using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true)) { var reader = new BinaryReader(stream); var count = reader.ReadInt32(); for (var i = 0; i < count; i++) { timeout.Delay(); var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader) { DateTimeKindHandling = DateTimeKind.Unspecified }); var metadata = doc.Value<RavenJObject>("@metadata"); if (metadata == null) throw new InvalidOperationException("Could not find metadata for document"); var id = metadata.Value<string>("@id"); if (string.IsNullOrEmpty(id)) throw new InvalidOperationException("Could not get id from metadata"); doc.Remove("@metadata"); yield return new JsonDocument { Key = id, DataAsJson = doc, Metadata = metadata }; } increaseDocumentsCount(count); } }
private HttpResponseMessage OnBulkOperation(Func<string, IndexQuery, BulkOperationOptions, RavenJArray> batchOperation, string index, CancellationTimeout timeout) { if (string.IsNullOrEmpty(index)) return GetEmptyMessage(HttpStatusCode.BadRequest); var option = new BulkOperationOptions { AllowStale = GetAllowStale(), MaxOpsPerSec = GetMaxOpsPerSec(), StaleTimeout = GetStaleTimeout(), RetrieveDetails = GetRetrieveDetails() }; var indexQuery = GetIndexQuery(maxPageSize: int.MaxValue); var status = new BulkOperationStatus(); long id; var task = Task.Factory.StartNew(() => { status.State = batchOperation(index, indexQuery, option); }).ContinueWith(t => { if (timeout != null) timeout.Dispose(); if (t.IsFaulted == false) { status.Completed = true; return; } var exception = t.Exception.ExtractSingleInnerException(); status.State = RavenJObject.FromObject(new { Error = exception.Message }); status.Faulted = true; status.Completed = true; }); Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.IndexBulkOperation, Payload = index }, out id, timeout.CancellationTokenSource); return GetMessageWithObject(new { OperationId = id }); }
public int BulkInsert(BulkInsertOptions options, IEnumerable<IEnumerable<JsonDocument>> docBatches, Guid operationId, CancellationToken token, CancellationTimeout timeout = null) { var documents = 0; Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification { OperationId = operationId, Type = DocumentChangeTypes.BulkInsertStarted }); using (var cts = CancellationTokenSource.CreateLinkedTokenSource(token, WorkContext.CancellationToken)) { foreach (var docs in docBatches) { cts.Token.ThrowIfCancellationRequested(); var docsToInsert = docs.ToArray(); var batch = 0; var keys = new HashSet<string>(StringComparer.OrdinalIgnoreCase); var collectionsAndEtags = new Dictionary<string, Etag>(StringComparer.OrdinalIgnoreCase); if (timeout != null) timeout.Pause(); using (Database.DocumentLock.Lock()) { if (timeout != null) timeout.Resume(); TransactionalStorage.Batch(accessor => { var inserts = 0; foreach (var doc in docsToInsert) { try { if (string.IsNullOrEmpty(doc.Key)) throw new InvalidOperationException("Cannot try to bulk insert a document without a key"); RemoveReservedProperties(doc.DataAsJson); RemoveMetadataReservedProperties(doc.Metadata); if (options.CheckReferencesInIndexes) keys.Add(doc.Key); documents++; batch++; AssertPutOperationNotVetoed(doc.Key, doc.Metadata, doc.DataAsJson, null); if (options.OverwriteExisting && options.SkipOverwriteIfUnchanged) { var existingDoc = accessor.Documents.DocumentByKey(doc.Key); if (IsTheSameDocument(doc, existingDoc)) continue; } foreach (var trigger in Database.PutTriggers) { trigger.Value.OnPut(doc.Key, doc.DataAsJson, doc.Metadata, null); } var result = accessor.Documents.InsertDocument(doc.Key, doc.DataAsJson, doc.Metadata, options.OverwriteExisting); if (result.Updated == false) inserts++; doc.Etag = result.Etag; doc.Metadata.EnsureSnapshot( "Metadata was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?"); doc.DataAsJson.EnsureSnapshot( "Document was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?"); var entityName = doc.Metadata.Value<string>(Constants.RavenEntityName); Etag highestEtagInCollection; if (string.IsNullOrEmpty(entityName) == false && (collectionsAndEtags.TryGetValue(entityName, out highestEtagInCollection) == false || result.Etag.CompareTo(highestEtagInCollection) > 0)) { collectionsAndEtags[entityName] = result.Etag; } foreach (var trigger in Database.PutTriggers) { trigger.Value.AfterPut(doc.Key, doc.DataAsJson, doc.Metadata, result.Etag, null); } Database.WorkContext.UpdateFoundWork(); } catch (Exception e) { Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification { OperationId = operationId, Message = e.Message, Etag = doc.Etag, Id = doc.Key, Type = DocumentChangeTypes.BulkInsertError }); throw; } } if (options.CheckReferencesInIndexes) { foreach (var key in keys) { Database.Indexes.CheckReferenceBecauseOfDocumentUpdate(key, accessor); } } accessor.Documents.IncrementDocumentCount(inserts); }); foreach (var collectionEtagPair in collectionsAndEtags) { Database.LastCollectionEtags.Update(collectionEtagPair.Key, collectionEtagPair.Value); } WorkContext.ShouldNotifyAboutWork(() => "BulkInsert batch of " + batch + " docs"); WorkContext.NotifyAboutWork(); // forcing notification so we would start indexing right away WorkContext.UpdateFoundWork(); } } } Database.Notifications.RaiseNotifications(new BulkInsertChangeNotification { OperationId = operationId, Type = DocumentChangeTypes.BulkInsertEnded }); if (documents > 0) WorkContext.ShouldNotifyAboutWork(() => "BulkInsert of " + documents + " docs"); return documents; }