public bool TryResolveIdenticalDocument(DocumentsOperationContext context, string id, BlittableJsonReaderObject incomingDoc, long lastModifiedTicks, string incomingChangeVector) { var existing = _database.DocumentsStorage.GetDocumentOrTombstone(context, id, throwOnConflict: false); var existingDoc = existing.Document; var existingTombstone = existing.Tombstone; if (existingDoc != null) { var compareResult = DocumentCompare.IsEqualTo(existingDoc.Data, incomingDoc, DocumentCompare.DocumentCompareOptions.MergeMetadata); if (compareResult == DocumentCompareResult.NotEqual) { return(false); } // no real conflict here, both documents have identical content so we only merge the change vector without increasing the local etag to prevent ping-pong replication var mergedChangeVector = ChangeVectorUtils.MergeVectors(incomingChangeVector, existingDoc.ChangeVector); var nonPersistentFlags = NonPersistentDocumentFlags.FromResolver; nonPersistentFlags |= compareResult.HasFlag(DocumentCompareResult.AttachmentsNotEqual) ? NonPersistentDocumentFlags.ResolveAttachmentsConflict : NonPersistentDocumentFlags.None; if (compareResult.HasFlag(DocumentCompareResult.CountersNotEqual)) { nonPersistentFlags |= NonPersistentDocumentFlags.ResolveCountersConflict; } _database.DocumentsStorage.Put(context, id, null, incomingDoc, lastModifiedTicks, mergedChangeVector, nonPersistentFlags: nonPersistentFlags); return(true); } if (existingTombstone != null && incomingDoc == null) { // Conflict between two tombstones resolves to the local tombstone existingTombstone.ChangeVector = ChangeVectorUtils.MergeVectors(incomingChangeVector, existingTombstone.ChangeVector); using (Slice.External(context.Allocator, existingTombstone.LowerId, out Slice lowerId)) { _database.DocumentsStorage.ConflictsStorage.DeleteConflicts(context, lowerId, null, existingTombstone.ChangeVector); } return(true); } return(false); }
private void HandleHiloConflict(DocumentsOperationContext context, string id, BlittableJsonReaderObject doc, string changeVector) { long highestMax; if (doc == null) { highestMax = 0; } else { if (!doc.TryGet("Max", out highestMax)) { throw new InvalidDataException("Tried to resolve HiLo document conflict but failed. Missing property name 'Max'"); } } var conflicts = _database.DocumentsStorage.ConflictsStorage.GetConflictsFor(context, id); var resolvedHiLoDoc = doc; string mergedChangeVector; if (conflicts.Count == 0) { //conflict with another existing document var localHiloDoc = _database.DocumentsStorage.Get(context, id); if (localHiloDoc.Data.TryGet("Max", out long max) && max > highestMax) { resolvedHiLoDoc = localHiloDoc.Data.Clone(context); } mergedChangeVector = ChangeVectorUtils.MergeVectors(changeVector, localHiloDoc.ChangeVector); } else { foreach (var conflict in conflicts) { if (conflict.Doc.TryGet("Max", out long tmpMax) && tmpMax > highestMax) { highestMax = tmpMax; resolvedHiLoDoc = conflict.Doc.Clone(context); } } var merged = ChangeVectorUtils.MergeVectors(conflicts.Select(c => c.ChangeVector).ToList()); mergedChangeVector = ChangeVectorUtils.MergeVectors(merged, changeVector); } _database.DocumentsStorage.Put(context, id, null, resolvedHiLoDoc, changeVector: mergedChangeVector, nonPersistentFlags: NonPersistentDocumentFlags.FromResolver); }
public override int Execute(DocumentsOperationContext context) { if (string.IsNullOrEmpty(context.LastDatabaseChangeVector)) { context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); } var status = ChangeVectorUtils.GetConflictStatus(_replicationBatchReply.DatabaseChangeVector, context.LastDatabaseChangeVector); if (status != ConflictStatus.AlreadyMerged) { return(0); } var result = ChangeVectorUtils.TryUpdateChangeVector(_replicationBatchReply.NodeTag, _dbId, _replicationBatchReply.CurrentEtag, context.LastDatabaseChangeVector); if (result.IsValid) { if (context.LastReplicationEtagFrom == null) { context.LastReplicationEtagFrom = new Dictionary <string, long>(); } if (context.LastReplicationEtagFrom.ContainsKey(_replicationBatchReply.DatabaseId) == false) { context.LastReplicationEtagFrom[_replicationBatchReply.DatabaseId] = _replicationBatchReply.CurrentEtag; } context.LastDatabaseChangeVector = result.ChangeVector; context.Transaction.InnerTransaction.LowLevelTransaction.OnDispose += _ => { try { _trigger.Set(); } catch { // } }; } return(result.IsValid ? 1 : 0); }
public static ConflictStatus GetConflictStatusForDocument(DocumentsOperationContext context, string id, LazyStringValue remote, out string conflictingVector) { //tombstones also can be a conflict entry conflictingVector = null; var conflicts = context.DocumentDatabase.DocumentsStorage.ConflictsStorage.GetConflictsFor(context, id); if (conflicts.Count > 0) { foreach (var existingConflict in conflicts) { if (ChangeVectorUtils.GetConflictStatus(remote, existingConflict.ChangeVector) == ConflictStatus.Conflict) { conflictingVector = existingConflict.ChangeVector; return(ConflictStatus.Conflict); } } // this document will resolve the conflicts when putted return(ConflictStatus.Update); } var result = context.DocumentDatabase.DocumentsStorage.GetDocumentOrTombstone(context, id); string local; if (result.Document != null) { local = result.Document.ChangeVector; } else if (result.Tombstone != null) { local = result.Tombstone.ChangeVector; } else { return(ConflictStatus.Update); //document with 'id' doesn't exist locally, so just do PUT } var status = ChangeVectorUtils.GetConflictStatus(remote, local); if (status == ConflictStatus.Conflict) { conflictingVector = local; } return(status); }
private int ReplicatedPastInternalDestinations(HashSet <string> internalUrls, string changeVector) { var count = 0; foreach (var destination in _outgoing) { if (internalUrls.Contains(destination.Destination.Url) == false) { continue; } var conflictStatus = ChangeVectorUtils.GetConflictStatus(changeVector, destination.LastAcceptedChangeVector); if (conflictStatus == ConflictStatus.AlreadyMerged) { count++; } } return(count); }
public bool HasHigherChangeVector(DocumentsOperationContext context, Slice prefixSlice, string expectedChangeVector) { if (ConflictsCount == 0) { return(false); } var conflictsTable = context.Transaction.InnerTransaction.OpenTable(ConflictsSchema, ConflictsSlice); foreach (var tvr in conflictsTable.SeekForwardFrom(ConflictsSchema.Indexes[IdAndChangeVectorSlice], prefixSlice, 0, true)) { var changeVector = TableValueToChangeVector(context, (int)ConflictsTable.ChangeVector, ref tvr.Result.Reader); if (ChangeVectorUtils.GetConflictStatus(changeVector, expectedChangeVector) == ConflictStatus.AlreadyMerged) { return(true); } } return(false); }
private bool ShouldSkip(ReplicationBatchItem item, OutgoingReplicationStatsScope stats, SkippedReplicationItemsInfo skippedReplicationItemsInfo) { switch (item) { case DocumentReplicationItem doc: if (doc.Flags.Contain(DocumentFlags.Artificial)) { stats.RecordArtificialDocumentSkip(); skippedReplicationItemsInfo.Update(item, isArtificial: true); return(true); } if (doc.Flags.Contain(DocumentFlags.Revision) || doc.Flags.Contain(DocumentFlags.DeleteRevision)) { // we let pass all the conflicted/resolved revisions, since we keep them with their original change vector which might be `AlreadyMerged` at the destination. if (doc.Flags.Contain(DocumentFlags.Conflicted) || doc.Flags.Contain(DocumentFlags.Resolved)) { return(false); } } break; case AttachmentReplicationItem _: if (MissingAttachmentsInLastBatch) { return(false); } break; } // destination already has it if (ChangeVectorUtils.GetConflictStatus(item.ChangeVector, _parent.LastAcceptedChangeVector) == ConflictStatus.AlreadyMerged) { stats.RecordChangeVectorSkip(); skippedReplicationItemsInfo.Update(item); return(true); } return(false); }
public string GetMergedConflictChangeVectorsAndDeleteConflicts(DocumentsOperationContext context, Slice lowerId, long newEtag, string existingChangeVector = null) { if (ConflictsCount == 0) { return(MergeVectorsWithoutConflicts(newEtag, existingChangeVector)); } var conflictChangeVectors = DeleteConflictsFor(context, lowerId, null).ChangeVectors; if (conflictChangeVectors == null || conflictChangeVectors.Count == 0) { return(MergeVectorsWithoutConflicts(newEtag, existingChangeVector)); } var newChangeVector = ChangeVectorUtils.NewChangeVector(_documentDatabase.ServerStore.NodeTag, newEtag, _documentsStorage.Environment.Base64Id); conflictChangeVectors.Add(newChangeVector); return(ChangeVectorUtils.MergeVectors(conflictChangeVectors)); }
private bool AddReplicationItemToBatch(ReplicationBatchItem item, OutgoingReplicationStatsScope stats, SkippedReplicationItemsInfo skippedReplicationItemsInfo) { if (item.Type == ReplicationBatchItem.ReplicationItemType.Document || item.Type == ReplicationBatchItem.ReplicationItemType.DocumentTombstone) { if ((item.Flags & DocumentFlags.Artificial) == DocumentFlags.Artificial) { stats.RecordArtificialDocumentSkip(); skippedReplicationItemsInfo.Update(item, isArtificial: true); return(false); } } // destination already has it if ((MissingAttachmentsInLastBatch == false || item.Type != ReplicationBatchItem.ReplicationItemType.Attachment) && ChangeVectorUtils.GetConflictStatus(item.ChangeVector, _parent.LastAcceptedChangeVector) == ConflictStatus.AlreadyMerged) { stats.RecordChangeVectorSkip(); skippedReplicationItemsInfo.Update(item); return(false); } if (skippedReplicationItemsInfo.SkippedItems > 0) { if (_log.IsInfoEnabled) { var message = skippedReplicationItemsInfo.GetInfoForDebug(_parent.LastAcceptedChangeVector); _log.Info(message); } skippedReplicationItemsInfo.Reset(); } if (item.Type == ReplicationBatchItem.ReplicationItemType.Attachment) { _replicaAttachmentStreams[item.Base64Hash] = item; } Debug.Assert(item.Flags.Contain(DocumentFlags.Artificial) == false); _orderedReplicaItems.Add(item.Etag, item); return(true); }
public void Remote_change_vector_with_different_dbId_set_than_local_should_return_Conflict_at_conflict_status() { var dbIds = new List <string> { new string('1', 22), new string('2', 22), new string('3', 22) }; var local = new[] { new ChangeVectorEntry { DbId = dbIds[0], Etag = 10, NodeTag = 0 }, }; var remote = new[] { new ChangeVectorEntry { DbId = dbIds[1], Etag = 10, NodeTag = 0 } }; Assert.Equal(ConflictStatus.Conflict, ChangeVectorUtils.GetConflictStatus(remote.SerializeVector(), local.SerializeVector())); }
public (string ChangeVector, NonPersistentDocumentFlags NonPersistentFlags) MergeConflictChangeVectorIfNeededAndDeleteConflicts(string documentChangeVector, DocumentsOperationContext context, string id, long newEtag, BlittableJsonReaderObject document) { var result = DeleteConflictsFor(context, id, document); if (result.ChangeVectors == null || result.ChangeVectors.Count == 0) { return(documentChangeVector, result.NonPersistentFlags); } var changeVectorList = new List <string> { documentChangeVector, context.LastDatabaseChangeVector ?? GetDatabaseChangeVector(context), ChangeVectorUtils.NewChangeVector(_documentDatabase.ServerStore.NodeTag, newEtag, _documentsStorage.Environment.Base64Id) }; changeVectorList.AddRange(result.ChangeVectors); return(ChangeVectorUtils.MergeVectors(changeVectorList), result.NonPersistentFlags); }
private unsafe bool AddReplicationItemToBatch(ReplicationBatchItem item, OutgoingReplicationStatsScope stats) { if (item.Type == ReplicationBatchItem.ReplicationItemType.Document || item.Type == ReplicationBatchItem.ReplicationItemType.DocumentTombstone) { if ((item.Flags & DocumentFlags.Artificial) == DocumentFlags.Artificial) { stats.RecordArtificialDocumentSkip(); if (_log.IsInfoEnabled) { _log.Info($"Skipping replication of {item.Id} because it is an artificial document"); } return(false); } } // destination already has it if (ChangeVectorUtils.GetConflictStatus(item.ChangeVector, _parent.LastAcceptedChangeVector) == ConflictStatus.AlreadyMerged) { stats.RecordChangeVectorSkip(); if (_log.IsInfoEnabled) { _log.Info($"Skipping replication of {item.Type} '{item.Id}' because destination has a higher change vector. Current: {item.ChangeVector} < Destination: {_parent.LastAcceptedChangeVector} "); } return(false); } if (item.Type == ReplicationBatchItem.ReplicationItemType.Attachment) { _replicaAttachmentStreams[item.Base64Hash] = item; } Debug.Assert(item.Flags.Contain(DocumentFlags.Artificial) == false); _orderedReplicaItems.Add(item.Etag, item); return(true); }
public bool TryResolveIdenticalDocument(DocumentsOperationContext context, string id, BlittableJsonReaderObject incomingDoc, long lastModifiedTicks, string incomingChangeVector) { var existing = _database.DocumentsStorage.GetDocumentOrTombstone(context, id, throwOnConflict: false); var existingDoc = existing.Document; var existingTombstone = existing.Tombstone; if (existingDoc != null) { var compareResult = DocumentCompare.IsEqualTo(existingDoc.Data, incomingDoc, true); if (compareResult == DocumentCompareResult.NotEqual) { return(false); } // no real conflict here, both documents have identical content var mergedChangeVector = ChangeVectorUtils.MergeVectors(incomingChangeVector, existingDoc.ChangeVector); var nonPersistentFlags = (compareResult & DocumentCompareResult.ShouldRecreateDocument) == DocumentCompareResult.ShouldRecreateDocument ? NonPersistentDocumentFlags.ResolveAttachmentsConflict : NonPersistentDocumentFlags.None; _database.DocumentsStorage.Put(context, id, null, incomingDoc, lastModifiedTicks, mergedChangeVector, nonPersistentFlags: nonPersistentFlags); return(true); } if (existingTombstone != null && incomingDoc == null) { // Conflict between two tombstones resolves to the local tombstone existingTombstone.ChangeVector = ChangeVectorUtils.MergeVectors(incomingChangeVector, existingTombstone.ChangeVector); using (Slice.External(context.Allocator, existingTombstone.LowerId, out Slice lowerId)) { _database.DocumentsStorage.ConflictsStorage.DeleteConflicts(context, lowerId, null, existingTombstone.ChangeVector); } return(true); } return(false); }
public void EtagShouldNotOverflow() { var cv1 = "A:86865297-V8jm+M9QKkuvfEUTQBfOtA, " + "C:87142328-5j4moMb8A0KxxcL9GhY/nw, " + "B:2146533895-SKM7aNMmSkW92wrQke+D4g, " + "E:1856361198-/mqfiL1AxkGlsqx1zwh2rw, " + "D:1882901489-TqJlheobc0KTcLDerIQ9oQ, " + "D:17267243-/3+4WZUBGkWL6/J4GMv2GA, " + "D:46103608-P1lQdjeAckGkdmY9RWr/Bg, " + "A:27850500-iUMDTgYwOkG25uod1g6gSg"; var cv2 = "C:87142328-5j4moMb8A0KxxcL9GhY/nw, " + "B:2146533895-SKM7aNMmSkW92wrQke+D4g, " + "E:1856361198-/mqfiL1AxkGlsqx1zwh2rw, " + "D:1882901489-TqJlheobc0KTcLDerIQ9oQ, " + "A:27850500-iUMDTgYwOkG25uod1g6gSg, " + "A:86865297-V8jm+M9QKkuvfEUTQBfOtA, " + "A:2319854662-eCGjjCNbP0CeTGSJMeqLZA"; ChangeVectorUtils.MergeVectors(cv1, cv2).ToChangeVector(); }
private long GetMinLastEtag() { var min = long.MaxValue; using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) { var record = _serverStore.Cluster.ReadRawDatabaseRecord(context, _database.Name); foreach (var taskId in record.GetPeriodicBackupsTaskIds()) { var config = record.GetPeriodicBackupConfiguration(taskId); if (config.IncrementalBackupFrequency == null) { continue; // if the backup is always full, we don't need to take into account the tombstones, since we never back them up. } var status = GetBackupStatusFromCluster(_serverStore, context, _database.Name, taskId); var etag = ChangeVectorUtils.GetEtagById(status.LastDatabaseChangeVector, _database.DbBase64Id); min = Math.Min(etag, min); } return(min); } }
public void Remote_change_vector_smaller_than_local_and_some_remote_etags_higher_than_local_should_return_Conflict_at_conflict_status() { var dbIds = new List <string> { new string('1', 22), new string('2', 22), new string('3', 22), new string('4', 22) }; var local = new[] { new ChangeVectorEntry { DbId = dbIds[0], Etag = 10, NodeTag = 0 }, new ChangeVectorEntry { DbId = dbIds[1], Etag = 20, NodeTag = 1 }, new ChangeVectorEntry { DbId = dbIds[2], Etag = 3000, NodeTag = 2 }, new ChangeVectorEntry { DbId = dbIds[3], Etag = 40, NodeTag = 3 } }; var remote = new[] { new ChangeVectorEntry { DbId = dbIds[0], Etag = 100, NodeTag = 0 }, new ChangeVectorEntry { DbId = dbIds[1], Etag = 200, NodeTag = 1 }, new ChangeVectorEntry { DbId = dbIds[2], Etag = 300, NodeTag = 2 } }; Assert.Equal(ConflictStatus.Conflict, ChangeVectorUtils.GetConflictStatus(remote.SerializeVector(), local.SerializeVector())); }
public void Remote_change_vector_larger_size_than_local_should_return_Update_at_conflict_status() { var dbIds = new List <string> { new string('1', 22), new string('2', 22), new string('3', 22) }; var local = new[] { new ChangeVectorEntry { DbId = dbIds[0], Etag = 10, NodeTag = 0 }, new ChangeVectorEntry { DbId = dbIds[1], Etag = 20, NodeTag = 1 }, new ChangeVectorEntry { DbId = dbIds[2], Etag = 30, NodeTag = 2 }, }; var remote = new[] { new ChangeVectorEntry { DbId = dbIds[0], Etag = 10, NodeTag = 0 }, new ChangeVectorEntry { DbId = dbIds[1], Etag = 20, NodeTag = 1 }, new ChangeVectorEntry { DbId = dbIds[2], Etag = 30, NodeTag = 2 }, new ChangeVectorEntry { DbId = dbIds[2], Etag = 40, NodeTag = 2 } }; Assert.Equal(ConflictStatus.Update, ChangeVectorUtils.GetConflictStatus(remote.SerializeVector(), local.SerializeVector())); }
public bool WaitForBiggerChangeVector(DocumentStore store, string changeVector) { var sw = Stopwatch.StartNew(); var timeout = 10000; if (Debugger.IsAttached) { timeout *= 10; } while (sw.ElapsedMilliseconds < timeout) { using (var session = store.OpenSession()) { var doc = session.Load <User>("users/1"); if (ChangeVectorUtils.GetConflictStatus(session.Advanced.GetChangeVectorFor(doc), changeVector) == ConflictStatus.Update) { return(true); } } Thread.Sleep(10); } return(false); }
private void PutCounterImpl(DocumentsOperationContext context, string documentId, string collection, string name, string changeVector, long value) { if (context.Transaction == null) { DocumentPutAction.ThrowRequiresTransaction(); Debug.Assert(false);// never hit } var collectionName = _documentsStorage.ExtractCollectionName(context, collection); var table = GetCountersTable(context.Transaction.InnerTransaction, collectionName); using (GetCounterKey(context, documentId, name, changeVector ?? context.Environment.Base64Id, out var counterKey)) { using (DocumentIdWorker.GetStringPreserveCase(context, name, out Slice nameSlice)) using (table.Allocate(out TableValueBuilder tvb)) { if (changeVector != null) { if (table.ReadByKey(counterKey, out var existing)) { var existingChangeVector = TableValueToChangeVector(context, (int)CountersTable.ChangeVector, ref existing); if (ChangeVectorUtils.GetConflictStatus(changeVector, existingChangeVector) == ConflictStatus.AlreadyMerged) { return; } } } RemoveTombstoneIfExists(context, documentId, name); var etag = _documentsStorage.GenerateNextEtag(); if (changeVector == null) { changeVector = ChangeVectorUtils .TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentsStorage.Environment.Base64Id, etag, string.Empty) .ChangeVector; } using (Slice.From(context.Allocator, changeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice)) { tvb.Add(counterKey); tvb.Add(nameSlice); tvb.Add(Bits.SwapBytes(etag)); tvb.Add(value); tvb.Add(cv); tvb.Add(collectionSlice); tvb.Add(context.TransactionMarkerOffset); table.Set(tvb); } UpdateMetrics(counterKey, name, changeVector, collection); context.Transaction.AddAfterCommitNotification(new CounterChange { ChangeVector = changeVector, DocumentId = documentId, Name = name, Value = value, Type = CounterChangeTypes.Put }); } } }
public PutOperationResults PutDocument(DocumentsOperationContext context, string id, string expectedChangeVector, BlittableJsonReaderObject document, long?lastModifiedTicks = null, string changeVector = null, DocumentFlags flags = DocumentFlags.None, NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None) { if (context.Transaction == null) { ThrowRequiresTransaction(); return(default(PutOperationResults)); // never hit } #if DEBUG var documentDebugHash = document.DebugHash; document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); #endif var newEtag = _documentsStorage.GenerateNextEtag(); var modifiedTicks = _documentsStorage.GetOrCreateLastModifiedTicks(lastModifiedTicks); id = BuildDocumentId(id, newEtag, out bool knownNewId); using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { var collectionName = _documentsStorage.ExtractCollectionName(context, document); var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents)); var oldValue = default(TableValueReader); if (knownNewId == false) { // delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size); table.ReadByKey(lowerId, out oldValue); } BlittableJsonReaderObject oldDoc = null; if (oldValue.Pointer == null) { // expectedChangeVector being null means we don't care, // and empty means that it must be new if (string.IsNullOrEmpty(expectedChangeVector) == false) { ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector); } } else { // expectedChangeVector has special meaning here // null - means, don't care, don't check // "" / empty - means, must be new // anything else - must match exactly if (expectedChangeVector != null) { var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue); if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0) { ThrowConcurrentException(id, expectedChangeVector, oldChangeVector); } } oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context); var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc); if (oldCollectionName != collectionName) { ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName); } var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue); if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate && (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication) { if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments) { flags |= DocumentFlags.HasAttachments; } } } var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue); if (string.IsNullOrEmpty(result.ChangeVector)) { ChangeVectorUtils.ThrowConflictingEtag(id, changeVector, newEtag, _documentsStorage.Environment.Base64Id, _documentDatabase.ServerStore.NodeTag); } changeVector = result.ChangeVector; nonPersistentFlags |= result.NonPersistentFlags; if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved)) { flags |= DocumentFlags.Resolved; } if (collectionName.IsHiLo == false && (flags & DocumentFlags.Artificial) != DocumentFlags.Artificial) { if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags)) { #if DEBUG if (document.DebugHash != documentDebugHash) { throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " + "this is likely because you are trying to save a document that is already stored and was moved"); } #endif document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); #if DEBUG documentDebugHash = document.DebugHash; document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); AttachmentsStorage.AssertAttachments(document, flags); #endif } if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false && (flags.Contain(DocumentFlags.Resolved) || _documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null )) { var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document, ref flags, out RevisionsCollectionConfiguration configuration); if (shouldVersion) { _documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags, changeVector, modifiedTicks, configuration, collectionName); } } } using (Slice.From(context.Allocator, changeVector, out var cv)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(Bits.SwapBytes(newEtag)); tvb.Add(idPtr); tvb.Add(document.BasePointer, document.Size); tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(modifiedTicks); tvb.Add((int)flags); tvb.Add(context.GetTransactionMarker()); if (oldValue.Pointer == null) { table.Insert(tvb); } else { table.Update(oldValue.Id, tvb); } } if (collectionName.IsHiLo == false) { _documentsStorage.ExpirationStorage.Put(context, lowerId, document); } context.LastDatabaseChangeVector = changeVector; _documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1); _documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size); context.Transaction.AddAfterCommitNotification(new DocumentChange { ChangeVector = changeVector, CollectionName = collectionName.Name, Id = id, Type = DocumentChangeTypes.Put, }); #if DEBUG if (document.DebugHash != documentDebugHash) { throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " + "this is likely because you are trying to save a document that is already stored and was moved"); } document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); AttachmentsStorage.AssertAttachments(document, flags); #endif return(new PutOperationResults { Etag = newEtag, Id = id, Collection = collectionName, ChangeVector = changeVector, Flags = flags, LastModified = new DateTime(modifiedTicks) }); } }
private async Task ProcessSubscriptionAsync() { if (_logger.IsInfoEnabled) { _logger.Info( $"Starting processing documents for subscription {SubscriptionId} received from {TcpConnection.TcpClient.Client.RemoteEndPoint}"); } using (DisposeOnDisconnect) using (TcpConnection.DocumentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext docsContext)) using (RegisterForNotificationOnNewDocuments()) { var replyFromClientTask = GetReplyFromClientAsync(); string lastChangeVector = null; string subscriptionChangeVectorBeforeCurrentBatch = SubscriptionState.ChangeVectorForNextBatchStartingPoint; var startEtag = GetStartEtagForSubscription(docsContext, SubscriptionState); var patch = SetupFilterScript(); var fetcher = new SubscriptionDocumentsFetcher(TcpConnection.DocumentDatabase, _options.MaxDocsPerBatch, SubscriptionId, TcpConnection.TcpClient.Client.RemoteEndPoint); while (CancellationTokenSource.IsCancellationRequested == false) { bool anyDocumentsSentInCurrentIteration = false; var sendingCurrentBatchStopwatch = Stopwatch.StartNew(); _buffer.SetLength(0); var docsToFlush = 0; using (TcpConnection.ContextPool.AllocateOperationContext(out JsonOperationContext context)) using (var writer = new BlittableJsonTextWriter(context, _buffer)) { using (docsContext.OpenReadTransaction()) { foreach (var result in fetcher.GetDataToSend(docsContext, Collection, Revisions, SubscriptionState, patch, startEtag)) { startEtag = result.Doc.Etag; lastChangeVector = string.IsNullOrEmpty(SubscriptionState.ChangeVectorForNextBatchStartingPoint) ? result.Doc.ChangeVector : ChangeVectorUtils.MergeVectors(result.Doc.ChangeVector, SubscriptionState.ChangeVectorForNextBatchStartingPoint); if (result.Doc.Data == null) { if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000) { await SendHeartBeat(); sendingCurrentBatchStopwatch.Restart(); } continue; } anyDocumentsSentInCurrentIteration = true; writer.WriteStartObject(); writer.WritePropertyName(context.GetLazyStringForFieldWithCaching(TypeSegment)); writer.WriteValue(BlittableJsonToken.String, context.GetLazyStringForFieldWithCaching(DataSegment)); writer.WriteComma(); writer.WritePropertyName(context.GetLazyStringForFieldWithCaching(DataSegment)); result.Doc.EnsureMetadata(); if (result.Exception != null) { var metadata = result.Doc.Data[Client.Constants.Documents.Metadata.Key]; writer.WriteValue(BlittableJsonToken.StartObject, docsContext.ReadObject(new DynamicJsonValue { [Client.Constants.Documents.Metadata.Key] = metadata }, result.Doc.Id) ); writer.WriteComma(); writer.WritePropertyName(context.GetLazyStringForFieldWithCaching(ExceptionSegment)); writer.WriteValue(BlittableJsonToken.String, context.GetLazyStringForFieldWithCaching(result.Exception.ToString())); } else { writer.WriteDocument(docsContext, result.Doc, metadataOnly: false); } writer.WriteEndObject(); docsToFlush++; // perform flush for current batch after 1000ms of running or 1 MB if (_buffer.Length > Constants.Size.Megabyte || sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000) { if (docsToFlush > 0) { await FlushDocsToClient(writer, docsToFlush); docsToFlush = 0; sendingCurrentBatchStopwatch.Restart(); } else { await SendHeartBeat(); } } } } if (anyDocumentsSentInCurrentIteration) { context.Write(writer, new DynamicJsonValue { [nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.EndOfBatch) }); await FlushDocsToClient(writer, docsToFlush, true); if (_logger.IsInfoEnabled) { _logger.Info( $"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}"); } } } if (anyDocumentsSentInCurrentIteration == false) { if (_logger.IsInfoEnabled) { _logger.Info( $"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}"); } await TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed(SubscriptionId, Options.SubscriptionName, lastChangeVector, subscriptionChangeVectorBeforeCurrentBatch); subscriptionChangeVectorBeforeCurrentBatch = lastChangeVector; if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000) { await SendHeartBeat(); } using (docsContext.OpenReadTransaction()) { long globalEtag = TcpConnection.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(docsContext, Collection); if (globalEtag > startEtag) { continue; } } if (await WaitForChangedDocuments(replyFromClientTask)) { continue; } } SubscriptionConnectionClientMessage clientReply; while (true) { var result = await Task.WhenAny(replyFromClientTask, TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(5000), CancellationTokenSource.Token)).ConfigureAwait(false); CancellationTokenSource.Token.ThrowIfCancellationRequested(); if (result == replyFromClientTask) { clientReply = await replyFromClientTask; if (clientReply.Type == SubscriptionConnectionClientMessage.MessageType.DisposedNotification) { CancellationTokenSource.Cancel(); break; } replyFromClientTask = GetReplyFromClientAsync(); break; } await SendHeartBeat(); } CancellationTokenSource.Token.ThrowIfCancellationRequested(); switch (clientReply.Type) { case SubscriptionConnectionClientMessage.MessageType.Acknowledge: await TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed( SubscriptionId, Options.SubscriptionName, lastChangeVector, subscriptionChangeVectorBeforeCurrentBatch); subscriptionChangeVectorBeforeCurrentBatch = lastChangeVector; Stats.LastAckReceivedAt = DateTime.UtcNow; Stats.AckRate.Mark(); await WriteJsonAsync(new DynamicJsonValue { [nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.Confirm) }); break; //precaution, should not reach this case... case SubscriptionConnectionClientMessage.MessageType.DisposedNotification: CancellationTokenSource.Cancel(); break; default: throw new ArgumentException("Unknown message type from client " + clientReply.Type); } } CancellationTokenSource.Token.ThrowIfCancellationRequested(); } }
public async Task DatabaseChangeVectorIsUpdatedCorrectly() { using (var store = GetDocumentStore()) { store.Maintenance.Send(new CreateSampleDataOperation()); var documentDatabase = (await Databases.GetDocumentDatabaseInstanceFor(store)); var documentsStorage = documentDatabase.DocumentsStorage; using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { var lastAttachmentChangeVector = string.Empty; var lastRevisionChangeVector = string.Empty; var attachmentStorage = documentDatabase.DocumentsStorage.AttachmentsStorage; foreach (var document in documentsStorage.GetDocumentsFrom(context, etag: 0)) { VerifyAttachments(); VerifyRevisions(); void VerifyAttachments() { if (document.Data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) == false) { return; } if (metadata.TryGet(Constants.Documents.Metadata.Attachments, out BlittableJsonReaderArray attachments) == false) { return; } foreach (BlittableJsonReaderObject attachmentFromDocument in attachments) { attachmentFromDocument.TryGet(nameof(AttachmentName.Name), out string attachmentName); var attachment = attachmentStorage.GetAttachment(context, document.Id, attachmentName, AttachmentType.Document, null); Assert.True(document.Etag > attachment.Etag); var conflictStatus = ChangeVectorUtils.GetConflictStatus(document.ChangeVector, attachment.ChangeVector); Assert.Equal(ConflictStatus.Update, conflictStatus); var attachmentConflictStatus = ChangeVectorUtils.GetConflictStatus(attachment.ChangeVector, lastAttachmentChangeVector); if (attachmentConflictStatus == ConflictStatus.Update) { lastAttachmentChangeVector = attachment.ChangeVector; } } } void VerifyRevisions() { var revisions = documentsStorage.RevisionsStorage.GetRevisions(context, document.Id, 0, int.MaxValue); foreach (var revision in revisions.Revisions) { var conflictStatus = ChangeVectorUtils.GetConflictStatus(revision.ChangeVector, lastRevisionChangeVector); if (conflictStatus == ConflictStatus.Update) { lastRevisionChangeVector = revision.ChangeVector; } } } } var lastChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); var lastCvAttachmentConflictStatus = ChangeVectorUtils.GetConflictStatus(lastAttachmentChangeVector, lastChangeVector); Assert.Equal(ConflictStatus.AlreadyMerged, lastCvAttachmentConflictStatus); var lastCvRevisionConflictStatus = ChangeVectorUtils.GetConflictStatus(lastRevisionChangeVector, lastChangeVector); Assert.Equal(ConflictStatus.AlreadyMerged, lastCvRevisionConflictStatus); } } }
public void AddConflict( DocumentsOperationContext context, string id, long lastModifiedTicks, BlittableJsonReaderObject incomingDoc, string incomingChangeVector, string incomingTombstoneCollection, DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None) { if (_logger.IsInfoEnabled) { _logger.Info($"Adding conflict to {id} (Incoming change vector {incomingChangeVector})"); } var tx = context.Transaction.InnerTransaction; var conflictsTable = tx.OpenTable(ConflictsSchema, ConflictsSlice); var fromSmuggler = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler; using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { CollectionName collectionName; // ReSharper disable once ArgumentsStyleLiteral var existing = _documentsStorage.GetDocumentOrTombstone(context, id, throwOnConflict: false); if (existing.Document != null) { var existingDoc = existing.Document; if (fromSmuggler == false) { using (Slice.From(context.Allocator, existingDoc.ChangeVector, out Slice cv)) using (DocumentIdWorker.GetStringPreserveCase(context, CollectionName.GetLazyCollectionNameFrom(context, existingDoc.Data), out Slice collectionSlice)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv); tvb.Add(idPtr); tvb.Add(existingDoc.Data.BasePointer, existingDoc.Data.Size); tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag())); tvb.Add(collectionSlice); tvb.Add(existingDoc.LastModified.Ticks); tvb.Add((int)existingDoc.Flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } } // we delete the data directly, without generating a tombstone, because we have a // conflict instead _documentsStorage.EnsureLastEtagIsPersisted(context, existingDoc.Etag); collectionName = _documentsStorage.ExtractCollectionName(context, existingDoc.Data); //make sure that the relevant collection tree exists var table = tx.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents)); table.Delete(existingDoc.StorageId); } else if (existing.Tombstone != null) { var existingTombstone = existing.Tombstone; if (fromSmuggler == false) { using (Slice.From(context.Allocator, existingTombstone.ChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, existingTombstone.Collection, out Slice collectionSlice)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv); tvb.Add(idPtr); tvb.Add(null, 0); tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag())); tvb.Add(collectionSlice); tvb.Add(existingTombstone.LastModified.Ticks); tvb.Add((int)existingTombstone.Flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } } // we delete the data directly, without generating a tombstone, because we have a // conflict instead _documentsStorage.EnsureLastEtagIsPersisted(context, existingTombstone.Etag); collectionName = _documentsStorage.GetCollection(existingTombstone.Collection, throwIfDoesNotExist: true); var table = tx.OpenTable(TombstonesSchema, collectionName.GetTableName(CollectionTableType.Tombstones)); table.Delete(existingTombstone.StorageId); } else // has existing conflicts { collectionName = _documentsStorage.ExtractCollectionName(context, incomingDoc); using (GetConflictsIdPrefix(context, lowerId, out Slice prefixSlice)) { var conflicts = GetConflictsFor(context, prefixSlice); foreach (var conflict in conflicts) { var conflictStatus = ChangeVectorUtils.GetConflictStatus(incomingChangeVector, conflict.ChangeVector); switch (conflictStatus) { case ConflictStatus.Update: DeleteConflictsFor(context, conflict.ChangeVector); // delete this, it has been subsumed break; case ConflictStatus.Conflict: if (fromSmuggler && DocumentCompare.IsEqualTo(conflict.Doc, incomingDoc, false) == DocumentCompareResult.Equal) { return; // we already have a conflict with equal content, no need to create another one } break; // we'll add this conflict if no one else also includes it case ConflictStatus.AlreadyMerged: return; // we already have a conflict that includes this version default: throw new ArgumentOutOfRangeException("Invalid conflict status " + conflictStatus); } } } } var etag = _documentsStorage.GenerateNextEtag(); if (context.LastDatabaseChangeVector == null) { context.LastDatabaseChangeVector = GetDatabaseChangeVector(context); } var result = ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentDatabase.DbBase64Id, etag, context.LastDatabaseChangeVector); if (result.IsValid) { context.LastDatabaseChangeVector = result.ChangeVector; } byte * doc = null; var docSize = 0; string collection; if (incomingDoc != null) // can be null if it is a tombstone { doc = incomingDoc.BasePointer; docSize = incomingDoc.Size; collection = CollectionName.GetLazyCollectionNameFrom(context, incomingDoc); } else { collection = incomingTombstoneCollection; } using (Slice.From(context.Allocator, incomingChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, collection, out Slice collectionSlice)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv); tvb.Add(idPtr); tvb.Add(doc, docSize); tvb.Add(Bits.SwapBytes(etag)); tvb.Add(collectionSlice); tvb.Add(lastModifiedTicks); tvb.Add((int)flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } context.Transaction.AddAfterCommitNotification(new DocumentChange { ChangeVector = incomingChangeVector, CollectionName = collectionName.Name, Id = id, Type = DocumentChangeTypes.Conflict, }); } }
private bool AboutToReadWithStateUnlikely(IJsonParser reader, JsonParserState state) { switch (_state) { case State.None: break; case State.IgnoreProperty: if (reader.Read() == false) { return(false); } if (state.CurrentTokenType == JsonParserToken.StartArray || state.CurrentTokenType == JsonParserToken.StartObject) { ThrowInvalidMetadataProperty(state); } break; case State.IgnoreArray: if (_verifyStartArray) { if (reader.Read() == false) { return(false); } _verifyStartArray = false; if (state.CurrentTokenType != JsonParserToken.StartArray) { ThrowInvalidReplicationHistoryType(state); } } while (state.CurrentTokenType != JsonParserToken.EndArray) { if (reader.Read() == false) { return(false); } } break; case State.IgnoreRevisionStatusProperty: if (reader.Read() == false) { return(false); } if (state.CurrentTokenType != JsonParserToken.String && state.CurrentTokenType != JsonParserToken.Integer) { ThrowInvalidEtagType(state); } switch (CreateLazyStringValueFromParserState(state)) { case LegacyHasRevisionsDocumentState: NonPersistentFlags |= NonPersistentDocumentFlags.LegacyHasRevisions; break; case LegacyRevisionState: NonPersistentFlags |= NonPersistentDocumentFlags.LegacyRevision; break; } break; case State.ReadingId: if (reader.Read() == false) { return(false); } if (state.CurrentTokenType != JsonParserToken.String) { ThrowExpectedFieldTypeOfString(Constants.Documents.Metadata.Id, state); } Id = CreateLazyStringValueFromParserState(state); break; case State.ReadingFlags: if (reader.Read() == false) { return(false); } if (state.CurrentTokenType != JsonParserToken.String) { ThrowExpectedFieldTypeOfString(Constants.Documents.Metadata.Flags, state); } Flags = ReadFlags(state); break; case State.ReadingChangeVector: if (reader.Read() == false) { return(false); } if (state.CurrentTokenType != JsonParserToken.String) { ThrowExpectedFieldTypeOfString(Constants.Documents.Metadata.ChangeVector, state); } ChangeVector = CreateLazyStringValueFromParserState(state); break; case State.ReadingFirstEtagOfLegacyRevision: if (reader.Read() == false) { return(false); } if (state.CurrentTokenType != JsonParserToken.String) { ThrowExpectedFieldTypeOfString("@etag", state); } _firstEtagOfLegacyRevision = CreateLazyStringValueFromParserState(state); ChangeVector = ChangeVectorUtils.NewChangeVector("RV", ++_legacyRevisionsCount, new Guid(_firstEtagOfLegacyRevision)); break; } return(true); }
private unsafe bool AboutToReadPropertyNameInMetadataUnlikely(IJsonParser reader, JsonParserState state, out bool aboutToReadPropertyName) { aboutToReadPropertyName = true; switch (state.StringSize) { default: // accept this property { return(true); } case -1: // IgnoreProperty { if (reader.Read() == false) { _state = State.IgnoreProperty; { aboutToReadPropertyName = false; return(true); } } if (state.CurrentTokenType == JsonParserToken.StartArray || state.CurrentTokenType == JsonParserToken.StartObject) { ThrowInvalidMetadataProperty(state); } break; } case 3: // @id if (state.StringBuffer[0] != (byte)'@' || *(short *)(state.StringBuffer + 1) != 25705) { aboutToReadPropertyName = true; return(true); } if (reader.Read() == false) { _state = State.ReadingId; { aboutToReadPropertyName = false; return(true); } } if (state.CurrentTokenType != JsonParserToken.String) { ThrowExpectedFieldTypeOfString(Constants.Documents.Metadata.Id, state); } Id = CreateLazyStringValueFromParserState(state); break; case 5: // @etag if (state.StringBuffer[0] != (byte)'@' || *(int *)(state.StringBuffer + 1) != 1734440037) { aboutToReadPropertyName = true; return(true); } if (ReadFirstEtagOfLegacyRevision && (NonPersistentFlags & NonPersistentDocumentFlags.LegacyRevision) == NonPersistentDocumentFlags.LegacyRevision) { if (_firstEtagOfLegacyRevision == null) { if (reader.Read() == false) { _state = State.ReadingFirstEtagOfLegacyRevision; { aboutToReadPropertyName = false; return(true); } } if (state.CurrentTokenType != JsonParserToken.String) { ThrowExpectedFieldTypeOfString("@etag", state); } _firstEtagOfLegacyRevision = CreateLazyStringValueFromParserState(state); ChangeVector = ChangeVectorUtils.NewChangeVector("RV", ++_legacyRevisionsCount, new Guid(_firstEtagOfLegacyRevision)); break; } ChangeVector = ChangeVectorUtils.NewChangeVector("RV", ++_legacyRevisionsCount, new Guid(_firstEtagOfLegacyRevision)); } goto case -1; case 6: // @flags if (state.StringBuffer[0] != (byte)'@' || *(int *)(state.StringBuffer + 1) != 1734437990 || state.StringBuffer[1 + sizeof(int)] != (byte)'s') { aboutToReadPropertyName = true; return(true); } if (reader.Read() == false) { _state = State.ReadingFlags; { aboutToReadPropertyName = false; return(true); } } if (state.CurrentTokenType != JsonParserToken.String) { ThrowExpectedFieldTypeOfString(Constants.Documents.Metadata.Flags, state); } Flags = ReadFlags(state); break; case 12: // @index-score if (state.StringBuffer[0] != (byte)'@' || *(long *)(state.StringBuffer + 1) != 7166121427196997225 || *(short *)(state.StringBuffer + 1 + sizeof(long)) != 29295 || state.StringBuffer[1 + sizeof(long) + sizeof(short)] != (byte)'e') { aboutToReadPropertyName = true; return(true); } goto case -1; case 13: //Last-Modified if (*(long *)state.StringBuffer != 7237087983830262092 || *(int *)(state.StringBuffer + sizeof(long)) != 1701406313 || state.StringBuffer[12] != (byte)'d') { aboutToReadPropertyName = true; return(true); } goto case -1; case 14: if (state.StringBuffer[0] == (byte)'@') { // @change-vector if (*(long *)(state.StringBuffer + 1) == 8515573965335390307 && *(int *)(state.StringBuffer + 1 + sizeof(long)) == 1869898597 && state.StringBuffer[1 + sizeof(long) + sizeof(int)] == (byte)'r') { if (reader.Read() == false) { _state = State.ReadingChangeVector; { aboutToReadPropertyName = false; return(true); } } if (state.CurrentTokenType != JsonParserToken.String) { ThrowExpectedFieldTypeOfString(Constants.Documents.Metadata.ChangeVector, state); } ChangeVector = CreateLazyStringValueFromParserState(state); break; } // @last-modified if (*(long *)(state.StringBuffer + 1) == 7237123168202350956 && *(int *)(state.StringBuffer + 1 + sizeof(long)) == 1701406313 && state.StringBuffer[1 + sizeof(long) + sizeof(int)] == (byte)'d') { goto case -1; } } { aboutToReadPropertyName = true; return(true); } case 15: //Raven-Read-Only if (*(long *)state.StringBuffer != 7300947898092904786 || *(int *)(state.StringBuffer + sizeof(long)) != 1328374881 || *(short *)(state.StringBuffer + sizeof(long) + sizeof(int)) != 27758 || state.StringBuffer[14] != (byte)'y') { aboutToReadPropertyName = true; return(true); } goto case -1; case 17: //Raven-Entity-Name --> @collection if (*(long *)state.StringBuffer != 7945807069737017682 || *(long *)(state.StringBuffer + sizeof(long)) != 7881666780093245812 || state.StringBuffer[16] != (byte)'e') { aboutToReadPropertyName = true; return(true); } var collection = _metadataCollections; state.StringBuffer = collection.AllocatedMemoryData.Address; state.StringSize = collection.Size; { aboutToReadPropertyName = true; return(true); } case 19: //Raven-Last-Modified if (*(long *)state.StringBuffer != 7011028672080929106 || *(long *)(state.StringBuffer + sizeof(long)) != 7379539893622240371 || *(short *)(state.StringBuffer + sizeof(long) + sizeof(long)) != 25961 || state.StringBuffer[18] != (byte)'d') { aboutToReadPropertyName = true; return(true); } goto case -1; case 21: //Raven-Expiration-Date if (*(long *)state.StringBuffer != 8666383010116297042 || *(long *)(state.StringBuffer + sizeof(long)) != 7957695015158966640 || *(short *)(state.StringBuffer + sizeof(long) + sizeof(long)) != 17453 || state.StringBuffer[20] != (byte)'e') { aboutToReadPropertyName = true; return(true); } var expires = _metadataExpires; state.StringBuffer = expires.AllocatedMemoryData.Address; state.StringSize = expires.Size; { aboutToReadPropertyName = true; return(true); } case 23: //Raven-Document-Revision if (*(long *)state.StringBuffer != 8017583188798234962 || *(long *)(state.StringBuffer + sizeof(long)) != 5921517102558967139 || *(int *)(state.StringBuffer + sizeof(long) + sizeof(long)) != 1936291429 || *(short *)(state.StringBuffer + sizeof(long) + sizeof(long) + sizeof(int)) != 28521 || state.StringBuffer[22] != (byte)'n') { aboutToReadPropertyName = true; return(true); } goto case -1; case 24: //Raven-Replication-Source if (*(long *)state.StringBuffer != 7300947898092904786 || *(long *)(state.StringBuffer + sizeof(long)) != 8028075772393122928 || *(long *)(state.StringBuffer + sizeof(long) + sizeof(long)) != 7305808869229538670) { aboutToReadPropertyName = true; return(true); } goto case -1; case 25: //Raven-Replication-Version OR Raven-Replication-History if (*(long *)state.StringBuffer != 7300947898092904786 || *(long *)(state.StringBuffer + sizeof(long)) != 8028075772393122928) { aboutToReadPropertyName = true; return(true); } var value = *(long *)(state.StringBuffer + sizeof(long) + sizeof(long)); var lastByte = state.StringBuffer[24]; if ((value != 8028074745928232302 || lastByte != (byte)'n') && (value != 8245937481775066478 || lastByte != (byte)'y')) { aboutToReadPropertyName = true; return(true); } var isReplicationHistory = lastByte == (byte)'y'; if (reader.Read() == false) { _verifyStartArray = isReplicationHistory; _state = isReplicationHistory ? State.IgnoreArray : State.IgnoreProperty; { aboutToReadPropertyName = false; return(true); } } // Raven-Replication-History is an array if (isReplicationHistory) { if (state.CurrentTokenType != JsonParserToken.StartArray) { ThrowInvalidReplicationHistoryType(state); } do { if (reader.Read() == false) { _state = State.IgnoreArray; { aboutToReadPropertyName = false; return(true); } } } while (state.CurrentTokenType != JsonParserToken.EndArray); } else if (state.CurrentTokenType == JsonParserToken.StartArray || state.CurrentTokenType == JsonParserToken.StartObject) { ThrowInvalidMetadataProperty(state); } break; case 29: //Non-Authoritative-Information if (*(long *)state.StringBuffer != 7526769800038477646 || *(long *)(state.StringBuffer + sizeof(long)) != 8532478930943832687 || *(long *)(state.StringBuffer + sizeof(long) + sizeof(long)) != 7886488383206796645 || *(int *)(state.StringBuffer + sizeof(long) + sizeof(long) + sizeof(long)) != 1869182049 || state.StringBuffer[28] != (byte)'n') { aboutToReadPropertyName = true; return(true); } goto case -1; case 30: //Raven-Document-Parent-Revision OR Raven-Document-Revision-Status if (*(long *)state.StringBuffer != 8017583188798234962) { aboutToReadPropertyName = true; return(true); } if ((*(long *)(state.StringBuffer + sizeof(long)) != 5777401914483111267 || *(long *)(state.StringBuffer + sizeof(long) + sizeof(long)) != 7300947924012593761 || *(int *)(state.StringBuffer + sizeof(long) + sizeof(long) + sizeof(long)) != 1769171318 || *(short *)(state.StringBuffer + sizeof(long) + sizeof(long) + sizeof(long) + sizeof(int)) != 28271) && (*(long *)(state.StringBuffer + sizeof(long)) != 5921517102558967139 || *(long *)(state.StringBuffer + sizeof(long) + sizeof(long)) != 3273676477843469925 || *(int *)(state.StringBuffer + sizeof(long) + sizeof(long) + sizeof(long)) != 1952543827 || *(short *)(state.StringBuffer + sizeof(long) + sizeof(long) + sizeof(long) + sizeof(int)) != 29557)) { aboutToReadPropertyName = true; return(true); } var isRevisionStatusProperty = state.StringBuffer[29] == 's'; if (reader.Read() == false) { _state = isRevisionStatusProperty ? State.IgnoreRevisionStatusProperty : State.IgnoreProperty; { aboutToReadPropertyName = false; return(true); } } if (state.CurrentTokenType == JsonParserToken.StartArray || state.CurrentTokenType == JsonParserToken.StartObject) { ThrowInvalidMetadataProperty(state); } if (isRevisionStatusProperty) { switch (CreateLazyStringValueFromParserState(state)) { case LegacyHasRevisionsDocumentState: NonPersistentFlags |= NonPersistentDocumentFlags.LegacyHasRevisions; break; case LegacyRevisionState: NonPersistentFlags |= NonPersistentDocumentFlags.LegacyRevision; break; } } break; case 32: //Raven-Replication-Merged-History if (*(long *)state.StringBuffer != 7300947898092904786 || *(long *)(state.StringBuffer + sizeof(long)) != 8028075772393122928 || *(long *)(state.StringBuffer + sizeof(long) + sizeof(long)) != 7234302117464059246 || *(long *)(state.StringBuffer + sizeof(long) + sizeof(long) + sizeof(long)) != 8751179571877464109) { aboutToReadPropertyName = true; return(true); } goto case -1; } return(false); }
public string IncrementCounter(DocumentsOperationContext context, string documentId, string collection, string name, long delta, out bool exists) { if (context.Transaction == null) { DocumentPutAction.ThrowRequiresTransaction(); Debug.Assert(false);// never hit } var collectionName = _documentsStorage.ExtractCollectionName(context, collection); var table = GetCountersTable(context.Transaction.InnerTransaction, collectionName); using (GetCounterKey(context, documentId, name, context.Environment.Base64Id, out var counterKey)) { var value = delta; exists = table.ReadByKey(counterKey, out var existing); if (exists) { var prev = *(long *)existing.Read((int)CountersTable.Value, out var size); Debug.Assert(size == sizeof(long)); try { value = checked (prev + delta); //inc } catch (OverflowException e) { CounterOverflowException.ThrowFor(documentId, name, prev, delta, e); } } RemoveTombstoneIfExists(context, documentId, name); var etag = _documentsStorage.GenerateNextEtag(); var result = ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentsStorage.Environment.Base64Id, etag, string.Empty); using (Slice.From(context.Allocator, result.ChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, name, out Slice nameSlice)) using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(counterKey); tvb.Add(nameSlice); tvb.Add(Bits.SwapBytes(etag)); tvb.Add(value); tvb.Add(cv); tvb.Add(collectionSlice); tvb.Add(context.TransactionMarkerOffset); table.Set(tvb); } UpdateMetrics(counterKey, name, result.ChangeVector, collection); context.Transaction.AddAfterCommitNotification(new CounterChange { ChangeVector = result.ChangeVector, DocumentId = documentId, Name = name, Type = exists ? CounterChangeTypes.Increment : CounterChangeTypes.Put, Value = value }); return(result.ChangeVector); } }
private void ReplicateToDestination() { try { AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiate); NativeMemory.EnsureRegistered(); if (_log.IsInfoEnabled) { _log.Info($"Will replicate to {Destination.FromString()} via {_connectionInfo.Url}"); } using (_parent._server.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) { var record = _parent.LoadDatabaseRecord(); if (record == null) { throw new InvalidOperationException($"The database record for {_parent.Database.Name} does not exist?!"); } if (record.Encrypted && Destination.Url.StartsWith("https:", StringComparison.OrdinalIgnoreCase) == false) { throw new InvalidOperationException( $"{record.DatabaseName} is encrypted, and require HTTPS for replication, but had endpoint with url {Destination.Url} to database {Destination.Database}"); } } var task = TcpUtils.ConnectSocketAsync(_connectionInfo, _parent._server.Engine.TcpConnectionTimeout, _log); task.Wait(CancellationToken); using (Interlocked.Exchange(ref _tcpClient, task.Result)) { var wrapSsl = TcpUtils.WrapStreamWithSslAsync(_tcpClient, _connectionInfo, _parent._server.Server.Certificate.Certificate, _parent._server.Engine.TcpConnectionTimeout); wrapSsl.Wait(CancellationToken); using (_stream = wrapSsl.Result) // note that _stream is being disposed by the interruptible read using (_interruptibleRead = new InterruptibleRead(_database.DocumentsStorage.ContextPool, _stream)) using (_buffer = JsonOperationContext.ManagedPinnedBuffer.LongLivedInstance()) { var documentSender = new ReplicationDocumentSender(_stream, this, _log); WriteHeaderToRemotePeer(); //handle initial response to last etag and staff try { var response = HandleServerResponse(getFullResponse: true); switch (response.ReplyType) { //The first time we start replication we need to register the destination current CV case ReplicationMessageReply.ReplyType.Ok: LastAcceptedChangeVector = response.Reply.DatabaseChangeVector; break; case ReplicationMessageReply.ReplyType.Error: var exception = new InvalidOperationException(response.Reply.Exception); if (response.Reply.Exception.Contains(nameof(DatabaseDoesNotExistException)) || response.Reply.Exception.Contains(nameof(DatabaseNotRelevantException))) { AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, "Database does not exist"); DatabaseDoesNotExistException.ThrowWithMessageAndException(Destination.Database, response.Reply.Message, exception); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, $"Got error: {response.Reply.Exception}"); throw exception; } } catch (DatabaseDoesNotExistException e) { var msg = $"Failed to parse initial server replication response, because there is no database named {_database.Name} " + "on the other end. "; if (_external) { msg += "In order for the replication to work, a database with the same name needs to be created at the destination"; } var young = (DateTime.UtcNow - _startedAt).TotalSeconds < 30; if (young) { msg += "This can happen if the other node wasn't yet notified about being assigned this database and should be resolved shortly."; } if (_log.IsInfoEnabled) { _log.Info(msg, e); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg); // won't add an alert on young connections // because it may take a few seconds for the other side to be notified by // the cluster that it has this db. if (young == false) { AddAlertOnFailureToReachOtherSide(msg, e); } throw; } catch (OperationCanceledException e) { const string msg = "Got operation canceled notification while opening outgoing replication channel. " + "Aborting and closing the channel."; if (_log.IsInfoEnabled) { _log.Info(msg, e); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg); throw; } catch (Exception e) { var msg = $"{OutgoingReplicationThreadName} got an unexpected exception during initial handshake"; if (_log.IsInfoEnabled) { _log.Info(msg, e); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg); AddAlertOnFailureToReachOtherSide(msg, e); throw; } DateTime nextReplicateAt = default(DateTime); while (_cts.IsCancellationRequested == false) { while (_database.Time.GetUtcNow() > nextReplicateAt) { if (_parent.DebugWaitAndRunReplicationOnce != null) { _parent.DebugWaitAndRunReplicationOnce.Wait(_cts.Token); _parent.DebugWaitAndRunReplicationOnce.Reset(); } var sp = Stopwatch.StartNew(); var stats = _lastStats = new OutgoingReplicationStatsAggregator(_parent.GetNextReplicationStatsId(), _lastStats); AddReplicationPerformance(stats); AddReplicationPulse(ReplicationPulseDirection.OutgoingBegin); try { using (var scope = stats.CreateScope()) { try { if (Destination is InternalReplication dest) { _parent.EnsureNotDeleted(dest.NodeTag); } var didWork = documentSender.ExecuteReplicationOnce(scope, ref nextReplicateAt); if (didWork == false) { break; } if (Destination is ExternalReplication externalReplication) { var taskId = externalReplication.TaskId; UpdateExternalReplicationInfo(taskId); } DocumentsSend?.Invoke(this); if (sp.ElapsedMilliseconds > 60 * 1000) { _waitForChanges.Set(); break; } } catch (OperationCanceledException) { // cancellation is not an actual error, // it is a "notification" that we need to cancel current operation const string msg = "Operation was canceled."; AddReplicationPulse(ReplicationPulseDirection.OutgoingError, msg); throw; } catch (Exception e) { AddReplicationPulse(ReplicationPulseDirection.OutgoingError, e.Message); scope.AddError(e); throw; } } } finally { stats.Complete(); AddReplicationPulse(ReplicationPulseDirection.OutgoingEnd); } } //if this returns false, this means either timeout or canceled token is activated while (WaitForChanges(_parent.MinimalHeartbeatInterval, _cts.Token) == false) { //If we got cancelled we need to break right away if (_cts.IsCancellationRequested) { break; } // open tx // read current change vector compare to last sent // if okay, send cv using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (var tx = ctx.OpenReadTransaction()) { var etag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction); if (etag == _lastSentDocumentEtag) { SendHeartbeat(DocumentsStorage.GetDatabaseChangeVector(ctx)); _parent.CompleteDeletionIfNeeded(); } else if (nextReplicateAt > DateTime.UtcNow) { SendHeartbeat(null); } else { //Send a heartbeat first so we will get an updated CV of the destination var currentChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); SendHeartbeat(null); //If our previous CV is already merged to the destination wait a bit more if (ChangeVectorUtils.GetConflictStatus(LastAcceptedChangeVector, currentChangeVector) == ConflictStatus.AlreadyMerged) { continue; } // we have updates that we need to send to the other side // let's do that.. // this can happen if we got replication from another node // that we need to send to it. Note that we typically // will wait for the other node to send the data directly to // our destination, but if it doesn't, we'll step in. // In this case, we try to limit congestion in the network and // only send updates that we have gotten from someone else after // a certain time, to let the other side tell us that it already // got it. Note that this is merely an optimization to reduce network // traffic. It is fine to have the same data come from different sources. break; } } } _waitForChanges.Reset(); } } } } catch (AggregateException e) { if (e.InnerExceptions.Count == 1) { if (e.InnerException is OperationCanceledException oce) { HandleOperationCancelException(oce); } if (e.InnerException is IOException ioe) { HandleIOException(ioe); } } HandleException(e); } catch (OperationCanceledException e) { HandleOperationCancelException(e); } catch (IOException e) { HandleIOException(e); } catch (Exception e) { HandleException(e); } void HandleOperationCancelException(OperationCanceledException e) { if (_log.IsInfoEnabled) { _log.Info($"Operation canceled on replication thread ({FromToString}). " + $"This is not necessary due to an issue. Stopped the thread."); } if (_cts.IsCancellationRequested == false) { Failed?.Invoke(this, e); } } void HandleIOException(IOException e) { if (_log.IsInfoEnabled) { if (e.InnerException is SocketException) { _log.Info($"SocketException was thrown from the connection to remote node ({FromToString}). " + $"This might mean that the remote node is done or there is a network issue.", e); } else { _log.Info($"IOException was thrown from the connection to remote node ({FromToString}).", e); } } Failed?.Invoke(this, e); } void HandleException(Exception e) { if (_log.IsInfoEnabled) { _log.Info($"Unexpected exception occurred on replication thread ({FromToString}). " + $"Replication stopped (will be retried later).", e); } Failed?.Invoke(this, e); } }
private bool AddReplicationItemToBatch(ReplicationBatchItem item, OutgoingReplicationStatsScope stats, SkippedReplicationItemsInfo skippedReplicationItemsInfo) { if (item.Type == ReplicationBatchItem.ReplicationItemType.Document || item.Type == ReplicationBatchItem.ReplicationItemType.DocumentTombstone) { if ((item.Flags & DocumentFlags.Artificial) == DocumentFlags.Artificial) { stats.RecordArtificialDocumentSkip(); skippedReplicationItemsInfo.Update(item, isArtificial: true); return(false); } } if (item.Type == ReplicationBatchItem.ReplicationItemType.CounterTombstone && _parent.SupportedFeatures.Replication.Counters == false) { // skip counter tombstones in legacy mode skippedReplicationItemsInfo.Update(item); return(false); } if (item.Flags.Contain(DocumentFlags.Revision) || item.Flags.Contain(DocumentFlags.DeleteRevision)) { // we let pass all the conflicted/resolved revisions, since we keep them with their original change vector which might be `AlreadyMerged` at the destination. if (item.Flags.Contain(DocumentFlags.Conflicted) || item.Flags.Contain(DocumentFlags.Resolved)) { _orderedReplicaItems.Add(item.Etag, item); return(true); } } // destination already has it if ((MissingAttachmentsInLastBatch == false || item.Type != ReplicationBatchItem.ReplicationItemType.Attachment) && ChangeVectorUtils.GetConflictStatus(item.ChangeVector, _parent.LastAcceptedChangeVector) == ConflictStatus.AlreadyMerged) { stats.RecordChangeVectorSkip(); skippedReplicationItemsInfo.Update(item); return(false); } if (skippedReplicationItemsInfo.SkippedItems > 0) { if (_log.IsInfoEnabled) { var message = skippedReplicationItemsInfo.GetInfoForDebug(_parent.LastAcceptedChangeVector); _log.Info(message); } skippedReplicationItemsInfo.Reset(); } if (item.Type == ReplicationBatchItem.ReplicationItemType.Attachment) { _replicaAttachmentStreams[item.Base64Hash] = item; } Debug.Assert(item.Flags.Contain(DocumentFlags.Artificial) == false); _orderedReplicaItems.Add(item.Etag, item); return(true); }
private void Delete(DocumentsOperationContext context, Slice lowerId, Slice idSlice, string id, CollectionName collectionName, BlittableJsonReaderObject deleteRevisionDocument, string changeVector, long lastModifiedTicks, NonPersistentDocumentFlags nonPersistentFlags, DocumentFlags flags) { Debug.Assert(changeVector != null, "Change vector must be set"); if (flags.Contain(DocumentFlags.HasAttachments)) { flags &= ~DocumentFlags.HasAttachments; } var configuration = GetRevisionsConfiguration(collectionName.Name, flags); if (configuration.Disabled) { return; } var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName); if (configuration.PurgeOnDelete) { using (GetKeyPrefix(context, lowerId, out Slice prefixSlice)) { DeleteRevisions(context, table, prefixSlice, collectionName, long.MaxValue, null, changeVector, lastModifiedTicks); DeleteCountOfRevisions(context, prefixSlice); } return; } var fromReplication = (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) == NonPersistentDocumentFlags.FromReplication; if (fromReplication) { void DeleteFromRevisionIfChangeVectorIsGreater() { TableValueReader tvr; try { var hasDoc = _documentsStorage.GetTableValueReaderForDocument(context, lowerId, throwOnConflict: true, tvr: out tvr); if (hasDoc == false) { return; } } catch (DocumentConflictException) { // Do not modify the document. return; } var docChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref tvr); if (ChangeVectorUtils.GetConflictStatus(changeVector, docChangeVector) == ConflictStatus.Update) { _documentsStorage.Delete(context, lowerId, id, null, lastModifiedTicks, changeVector, collectionName, nonPersistentFlags | NonPersistentDocumentFlags.FromRevision); } } DeleteFromRevisionIfChangeVectorIsGreater(); } var newEtag = _database.DocumentsStorage.GenerateNextEtag(); var newEtagSwapBytes = Bits.SwapBytes(newEtag); using (table.Allocate(out TableValueBuilder tvb)) using (Slice.From(context.Allocator, changeVector, out var cv)) { tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(newEtagSwapBytes); tvb.Add(idSlice); tvb.Add(deleteRevisionDocument.BasePointer, deleteRevisionDocument.Size); tvb.Add((int)(DocumentFlags.DeleteRevision | flags)); tvb.Add(newEtagSwapBytes); tvb.Add(lastModifiedTicks); tvb.Add(context.GetTransactionMarker()); if (flags.Contain(DocumentFlags.Resolved)) { tvb.Add((int)DocumentFlags.Resolved); } else { tvb.Add(0); } tvb.Add(Bits.SwapBytes(lastModifiedTicks)); var isNew = table.Set(tvb); if (isNew == false) { // It might be just an update from replication as we call this twice, both for the doc delete and for deleteRevision. return; } } DeleteOldRevisions(context, table, lowerId, collectionName, configuration, nonPersistentFlags, changeVector, lastModifiedTicks); }
public void Put(DocumentsOperationContext context, string id, BlittableJsonReaderObject document, DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags, string changeVector, long lastModifiedTicks, RevisionsCollectionConfiguration configuration = null, CollectionName collectionName = null) { Debug.Assert(changeVector != null, "Change vector must be set"); Debug.Assert(lastModifiedTicks != DateTime.MinValue.Ticks, "last modified ticks must be set"); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); if (collectionName == null) { collectionName = _database.DocumentsStorage.ExtractCollectionName(context, document); } using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { var fromSmuggler = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler; var fromReplication = (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) == NonPersistentDocumentFlags.FromReplication; var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName); // We want the revision's attachments to have a lower etag than the revision itself if ((flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments && fromSmuggler == false) { using (Slice.From(context.Allocator, changeVector, out Slice changeVectorSlice)) { if (table.VerifyKeyExists(changeVectorSlice) == false) { _documentsStorage.AttachmentsStorage.RevisionAttachments(context, lowerId, changeVectorSlice); } } } if (fromReplication) { void PutFromRevisionIfChangeVectorIsGreater() { bool hasDoc; TableValueReader tvr; try { hasDoc = _documentsStorage.GetTableValueReaderForDocument(context, lowerId, throwOnConflict: true, tvr: out tvr); } catch (DocumentConflictException) { // Do not modify the document. return; } if (hasDoc == false) { PutFromRevision(); return; } var docChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref tvr); if (ChangeVectorUtils.GetConflictStatus(changeVector, docChangeVector) == ConflictStatus.Update) { PutFromRevision(); } void PutFromRevision() { _documentsStorage.Put(context, id, null, document, lastModifiedTicks, changeVector, flags & ~DocumentFlags.Revision, nonPersistentFlags | NonPersistentDocumentFlags.FromRevision); } } PutFromRevisionIfChangeVectorIsGreater(); } flags |= DocumentFlags.Revision; var newEtag = _database.DocumentsStorage.GenerateNextEtag(); var newEtagSwapBytes = Bits.SwapBytes(newEtag); using (table.Allocate(out TableValueBuilder tvb)) using (Slice.From(context.Allocator, changeVector, out var cv)) { tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(newEtagSwapBytes); tvb.Add(idPtr); tvb.Add(document.BasePointer, document.Size); tvb.Add((int)flags); tvb.Add(NotDeletedRevisionMarker); tvb.Add(lastModifiedTicks); tvb.Add(context.GetTransactionMarker()); if (flags.Contain(DocumentFlags.Resolved)) { tvb.Add((int)DocumentFlags.Resolved); } else { tvb.Add(0); } tvb.Add(Bits.SwapBytes(lastModifiedTicks)); var isNew = table.Set(tvb); if (isNew == false) { // It might be just an update from replication as we call this twice, both for the doc delete and for deleteRevision. return; } } if (configuration == null) { configuration = GetRevisionsConfiguration(collectionName.Name); } DeleteOldRevisions(context, table, lowerId, collectionName, configuration, nonPersistentFlags, changeVector, lastModifiedTicks); } }