private void CreateTombstone(DocumentsOperationContext context, Slice keySlice, long revisionEtag, CollectionName collectionName, string changeVector, long lastModifiedTicks) { var newEtag = _documentsStorage.GenerateNextEtag(); var table = context.Transaction.InnerTransaction.OpenTable(TombstonesSchema, RevisionsTombstonesSlice); if (table.VerifyKeyExists(keySlice)) { return; // revisions (and revisions tombstones) are immutable, we can safely ignore this } using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice)) using (Slice.From(context.Allocator, changeVector, out var cv)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(keySlice.Content.Ptr, keySlice.Size); tvb.Add(Bits.SwapBytes(newEtag)); tvb.Add(Bits.SwapBytes(revisionEtag)); tvb.Add(context.GetTransactionMarker()); tvb.Add((byte)Tombstone.TombstoneType.Revision); tvb.Add(collectionSlice); tvb.Add((int)DocumentFlags.None); tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(lastModifiedTicks); table.Set(tvb); } }
public void Commit(DocumentsOperationContext txContext) { if (txContext != null) { if (txContext.GetTransactionMarker() == _lastCommittedTxMarker) { throw new InvalidOperationException( $"You're commiting transaction step with the same transaction marker as it was previously committed - {_lastCommittedTxMarker}. Did you forget to set it to write transaction id?"); } } _transactions.Commit(); if (txContext != null) { _lastCommittedTxMarker = txContext.GetTransactionMarker(); } }
public PutOperationResults PutDocument(DocumentsOperationContext context, string id, string expectedChangeVector, BlittableJsonReaderObject document, long?lastModifiedTicks = null, string changeVector = null, DocumentFlags flags = DocumentFlags.None, NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None) { if (context.Transaction == null) { ThrowRequiresTransaction(); return(default(PutOperationResults)); // never hit } #if DEBUG var documentDebugHash = document.DebugHash; document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); #endif var newEtag = _documentsStorage.GenerateNextEtag(); var modifiedTicks = lastModifiedTicks ?? _documentDatabase.Time.GetUtcNow().Ticks; id = BuildDocumentId(id, newEtag, out bool knownNewId); using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { var collectionName = _documentsStorage.ExtractCollectionName(context, document); var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents)); var oldValue = default(TableValueReader); if (knownNewId == false) { // delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size); table.ReadByKey(lowerId, out oldValue); } BlittableJsonReaderObject oldDoc = null; if (oldValue.Pointer == null) { // expectedChangeVector being null means we don't care, and empty // means that it must be new if (string.IsNullOrEmpty(expectedChangeVector) == false) { ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector); } } else { // expectedChangeVector has special meaning here // null - means, don't care, don't check // "" / empty - means, must be new // anything else - must match exactly if (expectedChangeVector != null) { var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue); if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0) { ThrowConcurrentException(id, expectedChangeVector, oldChangeVector); } } oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context); var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc); if (oldCollectionName != collectionName) { ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName); } var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue); if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate && (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication) { if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments) { flags |= DocumentFlags.HasAttachments; } } } var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue); changeVector = result.ChangeVector; nonPersistentFlags |= result.NonPersistentFlags; if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved)) { flags |= DocumentFlags.Resolved; } if (collectionName.IsHiLo == false && (flags & DocumentFlags.Artificial) != DocumentFlags.Artificial) { if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags)) { #if DEBUG if (document.DebugHash != documentDebugHash) { throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " + "this is likely because you are trying to save a document that is already stored and was moved"); } #endif document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); #if DEBUG documentDebugHash = document.DebugHash; document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); AttachmentsStorage.AssertAttachments(document, flags); #endif } if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false && (flags.Contain(DocumentFlags.Resolved) || _documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null )) { var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document, ref flags, out RevisionsCollectionConfiguration configuration); if (shouldVersion) { _documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags, changeVector, modifiedTicks, configuration, collectionName); } } } using (Slice.From(context.Allocator, changeVector, out var cv)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(Bits.SwapBytes(newEtag)); tvb.Add(idPtr); tvb.Add(document.BasePointer, document.Size); tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(modifiedTicks); tvb.Add((int)flags); tvb.Add(context.GetTransactionMarker()); if (oldValue.Pointer == null) { table.Insert(tvb); } else { table.Update(oldValue.Id, tvb); } } if (collectionName.IsHiLo == false) { _documentsStorage.ExpirationStorage.Put(context, lowerId, document); } context.LastDatabaseChangeVector = changeVector; _documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1); _documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size); context.Transaction.AddAfterCommitNotification(new DocumentChange { ChangeVector = changeVector, CollectionName = collectionName.Name, Id = id, Type = DocumentChangeTypes.Put, }); #if DEBUG if (document.DebugHash != documentDebugHash) { throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " + "this is likely because you are trying to save a document that is already stored and was moved"); } document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); AttachmentsStorage.AssertAttachments(document, flags); #endif return(new PutOperationResults { Etag = newEtag, Id = id, Collection = collectionName, ChangeVector = changeVector, Flags = flags, LastModified = new DateTime(modifiedTicks) }); } }
private void Delete(DocumentsOperationContext context, Slice lowerId, Slice idSlice, string id, CollectionName collectionName, BlittableJsonReaderObject deleteRevisionDocument, string changeVector, long lastModifiedTicks, NonPersistentDocumentFlags nonPersistentFlags, DocumentFlags flags) { Debug.Assert(changeVector != null, "Change vector must be set"); if (flags.Contain(DocumentFlags.HasAttachments)) { flags &= ~DocumentFlags.HasAttachments; } var configuration = GetRevisionsConfiguration(collectionName.Name, flags); if (configuration.Disabled) { return; } var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName); if (configuration.PurgeOnDelete) { using (GetKeyPrefix(context, lowerId, out Slice prefixSlice)) { DeleteRevisions(context, table, prefixSlice, collectionName, long.MaxValue, null, changeVector, lastModifiedTicks); DeleteCountOfRevisions(context, prefixSlice); } return; } var fromReplication = (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) == NonPersistentDocumentFlags.FromReplication; if (fromReplication) { void DeleteFromRevisionIfChangeVectorIsGreater() { TableValueReader tvr; try { var hasDoc = _documentsStorage.GetTableValueReaderForDocument(context, lowerId, throwOnConflict: true, tvr: out tvr); if (hasDoc == false) { return; } } catch (DocumentConflictException) { // Do not modify the document. return; } var docChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref tvr); if (ChangeVectorUtils.GetConflictStatus(changeVector, docChangeVector) == ConflictStatus.Update) { _documentsStorage.Delete(context, lowerId, id, null, lastModifiedTicks, changeVector, collectionName, nonPersistentFlags | NonPersistentDocumentFlags.FromRevision); } } DeleteFromRevisionIfChangeVectorIsGreater(); } var newEtag = _database.DocumentsStorage.GenerateNextEtag(); var newEtagSwapBytes = Bits.SwapBytes(newEtag); using (table.Allocate(out TableValueBuilder tvb)) using (Slice.From(context.Allocator, changeVector, out var cv)) { tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(newEtagSwapBytes); tvb.Add(idSlice); tvb.Add(deleteRevisionDocument.BasePointer, deleteRevisionDocument.Size); tvb.Add((int)(DocumentFlags.DeleteRevision | flags)); tvb.Add(newEtagSwapBytes); tvb.Add(lastModifiedTicks); tvb.Add(context.GetTransactionMarker()); if (flags.Contain(DocumentFlags.Resolved)) { tvb.Add((int)DocumentFlags.Resolved); } else { tvb.Add(0); } tvb.Add(Bits.SwapBytes(lastModifiedTicks)); var isNew = table.Set(tvb); if (isNew == false) { // It might be just an update from replication as we call this twice, both for the doc delete and for deleteRevision. return; } } DeleteOldRevisions(context, table, lowerId, collectionName, configuration, nonPersistentFlags, changeVector, lastModifiedTicks); }
public void Put(DocumentsOperationContext context, string id, BlittableJsonReaderObject document, DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags, string changeVector, long lastModifiedTicks, RevisionsCollectionConfiguration configuration = null, CollectionName collectionName = null) { Debug.Assert(changeVector != null, "Change vector must be set"); Debug.Assert(lastModifiedTicks != DateTime.MinValue.Ticks, "last modified ticks must be set"); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); if (collectionName == null) { collectionName = _database.DocumentsStorage.ExtractCollectionName(context, document); } using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { var fromSmuggler = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler; var fromReplication = (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) == NonPersistentDocumentFlags.FromReplication; var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName); // We want the revision's attachments to have a lower etag than the revision itself if ((flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments && fromSmuggler == false) { using (Slice.From(context.Allocator, changeVector, out Slice changeVectorSlice)) { if (table.VerifyKeyExists(changeVectorSlice) == false) { _documentsStorage.AttachmentsStorage.RevisionAttachments(context, lowerId, changeVectorSlice); } } } if (fromReplication) { void PutFromRevisionIfChangeVectorIsGreater() { bool hasDoc; TableValueReader tvr; try { hasDoc = _documentsStorage.GetTableValueReaderForDocument(context, lowerId, throwOnConflict: true, tvr: out tvr); } catch (DocumentConflictException) { // Do not modify the document. return; } if (hasDoc == false) { PutFromRevision(); return; } var docChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref tvr); if (ChangeVectorUtils.GetConflictStatus(changeVector, docChangeVector) == ConflictStatus.Update) { PutFromRevision(); } void PutFromRevision() { _documentsStorage.Put(context, id, null, document, lastModifiedTicks, changeVector, flags & ~DocumentFlags.Revision, nonPersistentFlags | NonPersistentDocumentFlags.FromRevision); } } PutFromRevisionIfChangeVectorIsGreater(); } flags |= DocumentFlags.Revision; var newEtag = _database.DocumentsStorage.GenerateNextEtag(); var newEtagSwapBytes = Bits.SwapBytes(newEtag); using (table.Allocate(out TableValueBuilder tvb)) using (Slice.From(context.Allocator, changeVector, out var cv)) { tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(newEtagSwapBytes); tvb.Add(idPtr); tvb.Add(document.BasePointer, document.Size); tvb.Add((int)flags); tvb.Add(NotDeletedRevisionMarker); tvb.Add(lastModifiedTicks); tvb.Add(context.GetTransactionMarker()); if (flags.Contain(DocumentFlags.Resolved)) { tvb.Add((int)DocumentFlags.Resolved); } else { tvb.Add(0); } tvb.Add(Bits.SwapBytes(lastModifiedTicks)); var isNew = table.Set(tvb); if (isNew == false) { // It might be just an update from replication as we call this twice, both for the doc delete and for deleteRevision. return; } } if (configuration == null) { configuration = GetRevisionsConfiguration(collectionName.Name); } DeleteOldRevisions(context, table, lowerId, collectionName, configuration, nonPersistentFlags, changeVector, lastModifiedTicks); } }
private CollectionName PutCounters(UpdateStep step, DocumentsOperationContext context, HashSet <string> dbIds, Dictionary <string, List <CounterDetail> > allCountersBatch, string docId) { string collection = null; using (DocumentIdWorker.GetSliceFromId(context, docId, out Slice lowerId)) { var docsTable = new Table(DocsSchema, step.ReadTx); if (docsTable.ReadByKey(lowerId, out var tvr)) { using (var doc = new BlittableJsonReaderObject(tvr.Read((int)DocumentsTable.Data, out int size), size, context)) { collection = CollectionName.GetCollectionName(doc); } } else { // document does not exist return(null); } } var collectionName = new CollectionName(collection); using (DocumentIdWorker.GetSliceFromId(context, docId, out Slice documentKeyPrefix, separator: SpecialChars.RecordSeparator)) { var maxNumberOfCountersPerGroup = Math.Max(32, 2048 / (dbIds.Count * 32 + 1)); // rough estimate var orderedKeys = allCountersBatch.OrderBy(x => x.Key).ToList(); var listOfDbIds = dbIds.ToList(); for (int i = 0; i < orderedKeys.Count / maxNumberOfCountersPerGroup + (orderedKeys.Count % maxNumberOfCountersPerGroup == 0 ? 0 : 1); i++) { var currentBatch = allCountersBatch.Skip(maxNumberOfCountersPerGroup * i).Take(maxNumberOfCountersPerGroup); using (var data = WriteNewCountersDocument(context, listOfDbIds, currentBatch)) { var etag = step.DocumentsStorage.GenerateNextEtag(); var changeVector = ChangeVectorUtils.NewChangeVector( step.DocumentsStorage.DocumentDatabase.ServerStore.NodeTag, etag, _dbId); var table = step.DocumentsStorage.CountersStorage.GetCountersTable(step.WriteTx, collectionName); data.TryGet(CountersStorage.Values, out BlittableJsonReaderObject values); BlittableJsonReaderObject.PropertyDetails prop = default; values.GetPropertyByIndex(0, ref prop); using (table.Allocate(out TableValueBuilder tvb)) { using (Slice.From(context.Allocator, changeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice)) using (context.Allocator.Allocate(documentKeyPrefix.Size + prop.Name.Size, out var counterKeyBuffer)) using (Slice.From(context.Allocator, prop.Name, out var nameSlice)) using (CreateCounterKeySlice(context, counterKeyBuffer, documentKeyPrefix, nameSlice, out var counterKeySlice)) { if (i == 0) { tvb.Add(documentKeyPrefix); } else { tvb.Add(counterKeySlice); } tvb.Add(Bits.SwapBytes(etag)); tvb.Add(cv); tvb.Add(data.BasePointer, data.Size); tvb.Add(collectionSlice); tvb.Add(context.GetTransactionMarker()); table.Set(tvb); } } } } } return(collectionName); }
private void DeleteCounter(UpdateStep step, LazyStringValue tombstoneKey, DocumentsOperationContext context) { var(docId, counterName) = ExtractDocIdAndNameFromCounterTombstone(context, tombstoneKey); using (docId) using (counterName) using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, docId, out Slice lowerId, out _)) { string collection = null; var docsTable = new Table(DocsSchema, step.ReadTx); if (docsTable.ReadByKey(lowerId, out var tvr)) { using (var doc = new BlittableJsonReaderObject(tvr.Read((int)DocumentsTable.Data, out int size), size, context)) { collection = CollectionName.GetCollectionName(doc); } } var collectionName = new CollectionName(collection); var table = step.DocumentsStorage.CountersStorage.GetCountersTable(step.WriteTx, collectionName); if (table.ReadByKey(lowerId, out var existing) == false) { return; } // (int)CountersTable.Data = 3 var data = new BlittableJsonReaderObject(existing.Read(3, out int oldSize), oldSize, context); if (data.TryGet(CountersStorage.Values, out BlittableJsonReaderObject counters) == false || counters.TryGetMember(counterName, out object counterToDelete) == false || counterToDelete is LazyStringValue) // already deleted { return; } var deleteCv = step.DocumentsStorage.CountersStorage.GenerateDeleteChangeVectorFromRawBlob(data, counterToDelete as BlittableJsonReaderObject.RawBlob); counters.Modifications = new DynamicJsonValue(counters) { [counterName] = deleteCv }; using (var old = data) { data = context.ReadObject(data, null, BlittableJsonDocumentBuilder.UsageMode.ToDisk); } var newEtag = step.DocumentsStorage.GenerateNextEtag(); var newChangeVector = ChangeVectorUtils.NewChangeVector(step.DocumentsStorage.DocumentDatabase.ServerStore.NodeTag, newEtag, _dbId); using (data) using (Slice.From(context.Allocator, newChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(Bits.SwapBytes(newEtag)); tvb.Add(cv); tvb.Add(data.BasePointer, data.Size); tvb.Add(collectionSlice); tvb.Add(context.GetTransactionMarker()); table.Set(tvb); } } }