public PutOperationResults PutDocument(DocumentsOperationContext context, string id, string expectedChangeVector, BlittableJsonReaderObject document, long?lastModifiedTicks = null, string changeVector = null, DocumentFlags flags = DocumentFlags.None, NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None) { if (context.Transaction == null) { ThrowRequiresTransaction(); return(default(PutOperationResults)); // never hit } #if DEBUG var documentDebugHash = document.DebugHash; document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); #endif var newEtag = _documentsStorage.GenerateNextEtag(); var modifiedTicks = lastModifiedTicks ?? _documentDatabase.Time.GetUtcNow().Ticks; id = BuildDocumentId(id, newEtag, out bool knownNewId); using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { var collectionName = _documentsStorage.ExtractCollectionName(context, document); var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents)); var oldValue = default(TableValueReader); if (knownNewId == false) { // delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size); table.ReadByKey(lowerId, out oldValue); } BlittableJsonReaderObject oldDoc = null; if (oldValue.Pointer == null) { // expectedChangeVector being null means we don't care, and empty // means that it must be new if (string.IsNullOrEmpty(expectedChangeVector) == false) { ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector); } } else { // expectedChangeVector has special meaning here // null - means, don't care, don't check // "" / empty - means, must be new // anything else - must match exactly if (expectedChangeVector != null) { var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue); if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0) { ThrowConcurrentException(id, expectedChangeVector, oldChangeVector); } } oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context); var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc); if (oldCollectionName != collectionName) { ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName); } var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue); if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate && (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication) { if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments) { flags |= DocumentFlags.HasAttachments; } } } var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue); changeVector = result.ChangeVector; nonPersistentFlags |= result.NonPersistentFlags; if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved)) { flags |= DocumentFlags.Resolved; } if (collectionName.IsHiLo == false && (flags & DocumentFlags.Artificial) != DocumentFlags.Artificial) { if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags)) { #if DEBUG if (document.DebugHash != documentDebugHash) { throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " + "this is likely because you are trying to save a document that is already stored and was moved"); } #endif document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); #if DEBUG documentDebugHash = document.DebugHash; document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); AttachmentsStorage.AssertAttachments(document, flags); #endif } if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false && (flags.Contain(DocumentFlags.Resolved) || _documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null )) { var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document, ref flags, out RevisionsCollectionConfiguration configuration); if (shouldVersion) { _documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags, changeVector, modifiedTicks, configuration, collectionName); } } } using (Slice.From(context.Allocator, changeVector, out var cv)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(Bits.SwapBytes(newEtag)); tvb.Add(idPtr); tvb.Add(document.BasePointer, document.Size); tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(modifiedTicks); tvb.Add((int)flags); tvb.Add(context.GetTransactionMarker()); if (oldValue.Pointer == null) { table.Insert(tvb); } else { table.Update(oldValue.Id, tvb); } } if (collectionName.IsHiLo == false) { _documentsStorage.ExpirationStorage.Put(context, lowerId, document); } context.LastDatabaseChangeVector = changeVector; _documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1); _documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size); context.Transaction.AddAfterCommitNotification(new DocumentChange { ChangeVector = changeVector, CollectionName = collectionName.Name, Id = id, Type = DocumentChangeTypes.Put, }); #if DEBUG if (document.DebugHash != documentDebugHash) { throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " + "this is likely because you are trying to save a document that is already stored and was moved"); } document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); AttachmentsStorage.AssertAttachments(document, flags); #endif return(new PutOperationResults { Etag = newEtag, Id = id, Collection = collectionName, ChangeVector = changeVector, Flags = flags, LastModified = new DateTime(modifiedTicks) }); } }
public string IncrementCounter(DocumentsOperationContext context, string documentId, string collection, string name, long delta, out bool exists) { if (context.Transaction == null) { DocumentPutAction.ThrowRequiresTransaction(); Debug.Assert(false);// never hit } var collectionName = _documentsStorage.ExtractCollectionName(context, collection); var table = GetCountersTable(context.Transaction.InnerTransaction, collectionName); using (GetCounterKey(context, documentId, name, context.Environment.Base64Id, out var counterKey)) { var value = delta; exists = table.ReadByKey(counterKey, out var existing); if (exists) { var prev = *(long *)existing.Read((int)CountersTable.Value, out var size); Debug.Assert(size == sizeof(long)); try { value = checked (prev + delta); //inc } catch (OverflowException e) { CounterOverflowException.ThrowFor(documentId, name, prev, delta, e); } } RemoveTombstoneIfExists(context, documentId, name); var etag = _documentsStorage.GenerateNextEtag(); var result = ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentsStorage.Environment.Base64Id, etag, string.Empty); using (Slice.From(context.Allocator, result.ChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, name, out Slice nameSlice)) using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(counterKey); tvb.Add(nameSlice); tvb.Add(Bits.SwapBytes(etag)); tvb.Add(value); tvb.Add(cv); tvb.Add(collectionSlice); tvb.Add(context.TransactionMarkerOffset); table.Set(tvb); } UpdateMetrics(counterKey, name, result.ChangeVector, collection); context.Transaction.AddAfterCommitNotification(new CounterChange { ChangeVector = result.ChangeVector, DocumentId = documentId, Name = name, Type = exists ? CounterChangeTypes.Increment : CounterChangeTypes.Put, Value = value }); return(result.ChangeVector); } }
public void AddConflict( DocumentsOperationContext context, string id, long lastModifiedTicks, BlittableJsonReaderObject incomingDoc, string incomingChangeVector, string incomingTombstoneCollection, DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None) { if (_logger.IsInfoEnabled) { _logger.Info($"Adding conflict to {id} (Incoming change vector {incomingChangeVector})"); } var tx = context.Transaction.InnerTransaction; var conflictsTable = tx.OpenTable(ConflictsSchema, ConflictsSlice); var fromSmuggler = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler; using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { CollectionName collectionName; // ReSharper disable once ArgumentsStyleLiteral var existing = _documentsStorage.GetDocumentOrTombstone(context, id, throwOnConflict: false); if (existing.Document != null) { var existingDoc = existing.Document; if (fromSmuggler == false) { using (Slice.From(context.Allocator, existingDoc.ChangeVector, out Slice cv)) using (DocumentIdWorker.GetStringPreserveCase(context, CollectionName.GetLazyCollectionNameFrom(context, existingDoc.Data), out Slice collectionSlice)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv); tvb.Add(idPtr); tvb.Add(existingDoc.Data.BasePointer, existingDoc.Data.Size); tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag())); tvb.Add(collectionSlice); tvb.Add(existingDoc.LastModified.Ticks); tvb.Add((int)existingDoc.Flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } } // we delete the data directly, without generating a tombstone, because we have a // conflict instead _documentsStorage.EnsureLastEtagIsPersisted(context, existingDoc.Etag); collectionName = _documentsStorage.ExtractCollectionName(context, existingDoc.Data); //make sure that the relevant collection tree exists var table = tx.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents)); table.Delete(existingDoc.StorageId); } else if (existing.Tombstone != null) { var existingTombstone = existing.Tombstone; if (fromSmuggler == false) { using (Slice.From(context.Allocator, existingTombstone.ChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, existingTombstone.Collection, out Slice collectionSlice)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv); tvb.Add(idPtr); tvb.Add(null, 0); tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag())); tvb.Add(collectionSlice); tvb.Add(existingTombstone.LastModified.Ticks); tvb.Add((int)existingTombstone.Flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } } // we delete the data directly, without generating a tombstone, because we have a // conflict instead _documentsStorage.EnsureLastEtagIsPersisted(context, existingTombstone.Etag); collectionName = _documentsStorage.GetCollection(existingTombstone.Collection, throwIfDoesNotExist: true); var table = tx.OpenTable(TombstonesSchema, collectionName.GetTableName(CollectionTableType.Tombstones)); table.Delete(existingTombstone.StorageId); } else // has existing conflicts { collectionName = _documentsStorage.ExtractCollectionName(context, incomingDoc); using (GetConflictsIdPrefix(context, lowerId, out Slice prefixSlice)) { var conflicts = GetConflictsFor(context, prefixSlice); foreach (var conflict in conflicts) { var conflictStatus = ChangeVectorUtils.GetConflictStatus(incomingChangeVector, conflict.ChangeVector); switch (conflictStatus) { case ConflictStatus.Update: DeleteConflictsFor(context, conflict.ChangeVector); // delete this, it has been subsumed break; case ConflictStatus.Conflict: if (fromSmuggler && DocumentCompare.IsEqualTo(conflict.Doc, incomingDoc, false) == DocumentCompareResult.Equal) { return; // we already have a conflict with equal content, no need to create another one } break; // we'll add this conflict if no one else also includes it case ConflictStatus.AlreadyMerged: return; // we already have a conflict that includes this version default: throw new ArgumentOutOfRangeException("Invalid conflict status " + conflictStatus); } } } } var etag = _documentsStorage.GenerateNextEtag(); if (context.LastDatabaseChangeVector == null) { context.LastDatabaseChangeVector = GetDatabaseChangeVector(context); } var result = ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentDatabase.DbBase64Id, etag, context.LastDatabaseChangeVector); if (result.IsValid) { context.LastDatabaseChangeVector = result.ChangeVector; } byte * doc = null; var docSize = 0; string collection; if (incomingDoc != null) // can be null if it is a tombstone { doc = incomingDoc.BasePointer; docSize = incomingDoc.Size; collection = CollectionName.GetLazyCollectionNameFrom(context, incomingDoc); } else { collection = incomingTombstoneCollection; } using (Slice.From(context.Allocator, incomingChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, collection, out Slice collectionSlice)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv); tvb.Add(idPtr); tvb.Add(doc, docSize); tvb.Add(Bits.SwapBytes(etag)); tvb.Add(collectionSlice); tvb.Add(lastModifiedTicks); tvb.Add((int)flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } context.Transaction.AddAfterCommitNotification(new DocumentChange { ChangeVector = incomingChangeVector, CollectionName = collectionName.Name, Id = id, Type = DocumentChangeTypes.Conflict, }); } }
private void PutCounterImpl(DocumentsOperationContext context, string documentId, string collection, string name, string changeVector, long value) { if (context.Transaction == null) { DocumentPutAction.ThrowRequiresTransaction(); Debug.Assert(false);// never hit } var collectionName = _documentsStorage.ExtractCollectionName(context, collection); var table = GetCountersTable(context.Transaction.InnerTransaction, collectionName); using (GetCounterKey(context, documentId, name, changeVector ?? context.Environment.Base64Id, out var counterKey)) { using (DocumentIdWorker.GetStringPreserveCase(context, name, out Slice nameSlice)) using (table.Allocate(out TableValueBuilder tvb)) { if (changeVector != null) { if (table.ReadByKey(counterKey, out var existing)) { var existingChangeVector = TableValueToChangeVector(context, (int)CountersTable.ChangeVector, ref existing); if (ChangeVectorUtils.GetConflictStatus(changeVector, existingChangeVector) == ConflictStatus.AlreadyMerged) { return; } } } RemoveTombstoneIfExists(context, documentId, name); var etag = _documentsStorage.GenerateNextEtag(); if (changeVector == null) { changeVector = ChangeVectorUtils .TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentsStorage.Environment.Base64Id, etag, string.Empty) .ChangeVector; } using (Slice.From(context.Allocator, changeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice)) { tvb.Add(counterKey); tvb.Add(nameSlice); tvb.Add(Bits.SwapBytes(etag)); tvb.Add(value); tvb.Add(cv); tvb.Add(collectionSlice); tvb.Add(context.TransactionMarkerOffset); table.Set(tvb); } UpdateMetrics(counterKey, name, changeVector, collection); context.Transaction.AddAfterCommitNotification(new CounterChange { ChangeVector = changeVector, DocumentId = documentId, Name = name, Value = value, Type = CounterChangeTypes.Put }); } } }