public override int Execute(DocumentsOperationContext context) { if (string.IsNullOrEmpty(context.LastDatabaseChangeVector)) { context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); } var status = ChangeVectorUtils.GetConflictStatus(_replicationBatchReply.DatabaseChangeVector, context.LastDatabaseChangeVector); if (status != ConflictStatus.AlreadyMerged) { return(0); } var res = ChangeVectorUtils.TryUpdateChangeVector(_replicationBatchReply.NodeTag, _dbId, _replicationBatchReply.CurrentEtag, ref context.LastDatabaseChangeVector) ? 1 : 0; if (res == 1) { context.Transaction.InnerTransaction.LowLevelTransaction.OnDispose += _ => { try { _trigger.Set(); } catch { // } }; } return(res); }
private string MergeVectorsWithoutConflicts(long newEtag, string existing) { if (existing != null) { var result = ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentsStorage.Environment.Base64Id, newEtag, existing); return(result.ChangeVector); } return(ChangeVectorUtils.NewChangeVector(_documentDatabase.ServerStore.NodeTag, newEtag, _documentsStorage.Environment.Base64Id)); }
private string SetDocumentChangeVectorForLocalChange(DocumentsOperationContext context, Slice lowerId, string oldChangeVector, long newEtag) { if (string.IsNullOrEmpty(oldChangeVector) == false) { ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentsStorage.Environment.DbId, newEtag, ref oldChangeVector); return(oldChangeVector); } return(_documentsStorage.ConflictsStorage.GetMergedConflictChangeVectorsAndDeleteConflicts(context, lowerId, newEtag)); }
public string IncrementCounter(DocumentsOperationContext context, string documentId, string collection, string name, long value) { if (context.Transaction == null) { DocumentPutAction.ThrowRequiresTransaction(); Debug.Assert(false);// never hit } var collectionName = _documentsStorage.ExtractCollectionName(context, collection); var table = GetCountersTable(context.Transaction.InnerTransaction, collectionName); using (GetCounterKey(context, documentId, name, context.Environment.Base64Id, out var counterKey)) { long prev = 0; if (table.ReadByKey(counterKey, out var existing)) { prev = *(long *)existing.Read((int)CountersTable.Value, out var size); Debug.Assert(size == sizeof(long)); } RemoveTombstoneIfExists(context, documentId, name); var etag = _documentsStorage.GenerateNextEtag(); var result = ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentsStorage.Environment.Base64Id, etag, string.Empty); using (Slice.From(context.Allocator, result.ChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, name, out Slice nameSlice)) using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(counterKey); tvb.Add(nameSlice); tvb.Add(Bits.SwapBytes(etag)); tvb.Add(prev + value); //inc tvb.Add(cv); tvb.Add(collectionSlice); tvb.Add(context.TransactionMarkerOffset); table.Set(tvb); } context.Transaction.AddAfterCommitNotification(new DocumentChange { ChangeVector = result.ChangeVector, Id = documentId, CounterName = name, Type = DocumentChangeTypes.Counter }); return(result.ChangeVector); } }
public override int Execute(DocumentsOperationContext context) { if (string.IsNullOrEmpty(context.LastDatabaseChangeVector)) { context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); } var status = ChangeVectorUtils.GetConflictStatus(_replicationBatchReply.DatabaseChangeVector, context.LastDatabaseChangeVector); if (status != ConflictStatus.AlreadyMerged) { return(0); } var result = ChangeVectorUtils.TryUpdateChangeVector(_replicationBatchReply.NodeTag, _dbId, _replicationBatchReply.CurrentEtag, context.LastDatabaseChangeVector); if (result.IsValid) { if (context.LastReplicationEtagFrom == null) { context.LastReplicationEtagFrom = new Dictionary <string, long>(); } if (context.LastReplicationEtagFrom.ContainsKey(_replicationBatchReply.DatabaseId) == false) { context.LastReplicationEtagFrom[_replicationBatchReply.DatabaseId] = _replicationBatchReply.CurrentEtag; } context.LastDatabaseChangeVector = result.ChangeVector; context.Transaction.InnerTransaction.LowLevelTransaction.OnDispose += _ => { try { _trigger.Set(); } catch { // } }; } return(result.IsValid ? 1 : 0); }
public void AddConflict( DocumentsOperationContext context, string id, long lastModifiedTicks, BlittableJsonReaderObject incomingDoc, string incomingChangeVector, string incomingTombstoneCollection, DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None) { if (_logger.IsInfoEnabled) { _logger.Info($"Adding conflict to {id} (Incoming change vector {incomingChangeVector})"); } var tx = context.Transaction.InnerTransaction; var conflictsTable = tx.OpenTable(ConflictsSchema, ConflictsSlice); var fromSmuggler = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler; using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { CollectionName collectionName; // ReSharper disable once ArgumentsStyleLiteral var existing = _documentsStorage.GetDocumentOrTombstone(context, id, throwOnConflict: false); if (existing.Document != null) { var existingDoc = existing.Document; if (fromSmuggler == false) { using (Slice.From(context.Allocator, existingDoc.ChangeVector, out Slice cv)) using (DocumentIdWorker.GetStringPreserveCase(context, CollectionName.GetLazyCollectionNameFrom(context, existingDoc.Data), out Slice collectionSlice)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv); tvb.Add(idPtr); tvb.Add(existingDoc.Data.BasePointer, existingDoc.Data.Size); tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag())); tvb.Add(collectionSlice); tvb.Add(existingDoc.LastModified.Ticks); tvb.Add((int)existingDoc.Flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } } // we delete the data directly, without generating a tombstone, because we have a // conflict instead _documentsStorage.EnsureLastEtagIsPersisted(context, existingDoc.Etag); collectionName = _documentsStorage.ExtractCollectionName(context, existingDoc.Data); //make sure that the relevant collection tree exists var table = tx.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents)); table.Delete(existingDoc.StorageId); } else if (existing.Tombstone != null) { var existingTombstone = existing.Tombstone; if (fromSmuggler == false) { using (Slice.From(context.Allocator, existingTombstone.ChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, existingTombstone.Collection, out Slice collectionSlice)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv); tvb.Add(idPtr); tvb.Add(null, 0); tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag())); tvb.Add(collectionSlice); tvb.Add(existingTombstone.LastModified.Ticks); tvb.Add((int)existingTombstone.Flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } } // we delete the data directly, without generating a tombstone, because we have a // conflict instead _documentsStorage.EnsureLastEtagIsPersisted(context, existingTombstone.Etag); collectionName = _documentsStorage.GetCollection(existingTombstone.Collection, throwIfDoesNotExist: true); var table = tx.OpenTable(TombstonesSchema, collectionName.GetTableName(CollectionTableType.Tombstones)); table.Delete(existingTombstone.StorageId); } else // has existing conflicts { collectionName = _documentsStorage.ExtractCollectionName(context, incomingDoc); using (GetConflictsIdPrefix(context, lowerId, out Slice prefixSlice)) { var conflicts = GetConflictsFor(context, prefixSlice); foreach (var conflict in conflicts) { var conflictStatus = ChangeVectorUtils.GetConflictStatus(incomingChangeVector, conflict.ChangeVector); switch (conflictStatus) { case ConflictStatus.Update: DeleteConflictsFor(context, conflict.ChangeVector); // delete this, it has been subsumed break; case ConflictStatus.Conflict: if (fromSmuggler && DocumentCompare.IsEqualTo(conflict.Doc, incomingDoc, false) == DocumentCompareResult.Equal) { return; // we already have a conflict with equal content, no need to create another one } break; // we'll add this conflict if no one else also includes it case ConflictStatus.AlreadyMerged: return; // we already have a conflict that includes this version default: throw new ArgumentOutOfRangeException("Invalid conflict status " + conflictStatus); } } } } var etag = _documentsStorage.GenerateNextEtag(); if (context.LastDatabaseChangeVector == null) { context.LastDatabaseChangeVector = GetDatabaseChangeVector(context); } var result = ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentDatabase.DbBase64Id, etag, context.LastDatabaseChangeVector); if (result.IsValid) { context.LastDatabaseChangeVector = result.ChangeVector; } byte * doc = null; var docSize = 0; string collection; if (incomingDoc != null) // can be null if it is a tombstone { doc = incomingDoc.BasePointer; docSize = incomingDoc.Size; collection = CollectionName.GetLazyCollectionNameFrom(context, incomingDoc); } else { collection = incomingTombstoneCollection; } using (Slice.From(context.Allocator, incomingChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, collection, out Slice collectionSlice)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv); tvb.Add(idPtr); tvb.Add(doc, docSize); tvb.Add(Bits.SwapBytes(etag)); tvb.Add(collectionSlice); tvb.Add(lastModifiedTicks); tvb.Add((int)flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } context.Transaction.AddAfterCommitNotification(new DocumentChange { ChangeVector = incomingChangeVector, CollectionName = collectionName.Name, Id = id, Type = DocumentChangeTypes.Conflict, }); } }
private void PutCounterImpl(DocumentsOperationContext context, string documentId, string collection, string name, string changeVector, long value) { if (context.Transaction == null) { DocumentPutAction.ThrowRequiresTransaction(); Debug.Assert(false);// never hit } var collectionName = _documentsStorage.ExtractCollectionName(context, collection); var table = GetCountersTable(context.Transaction.InnerTransaction, collectionName); using (GetCounterKey(context, documentId, name, changeVector ?? context.Environment.Base64Id, out var counterKey)) { using (DocumentIdWorker.GetStringPreserveCase(context, name, out Slice nameSlice)) using (table.Allocate(out TableValueBuilder tvb)) { if (changeVector != null) { if (table.ReadByKey(counterKey, out var existing)) { var existingChangeVector = TableValueToChangeVector(context, (int)CountersTable.ChangeVector, ref existing); if (ChangeVectorUtils.GetConflictStatus(changeVector, existingChangeVector) == ConflictStatus.AlreadyMerged) { return; } } } RemoveTombstoneIfExists(context, documentId, name); var etag = _documentsStorage.GenerateNextEtag(); if (changeVector == null) { changeVector = ChangeVectorUtils .TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentsStorage.Environment.Base64Id, etag, string.Empty) .ChangeVector; } using (Slice.From(context.Allocator, changeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice)) { tvb.Add(counterKey); tvb.Add(nameSlice); tvb.Add(Bits.SwapBytes(etag)); tvb.Add(value); tvb.Add(cv); tvb.Add(collectionSlice); tvb.Add(context.TransactionMarkerOffset); table.Set(tvb); } UpdateMetrics(counterKey, name, changeVector, collection); context.Transaction.AddAfterCommitNotification(new CounterChange { ChangeVector = changeVector, DocumentId = documentId, Name = name, Value = value, Type = CounterChangeTypes.Put }); } } }
public (List <string> ChangeVectors, NonPersistentDocumentFlags NonPersistentFlags) DeleteConflictsFor( DocumentsOperationContext context, Slice lowerId, BlittableJsonReaderObject document) { if (ConflictsCount == 0) { return(null, NonPersistentDocumentFlags.None); } var changeVectors = new List <string>(); var nonPersistentFlags = NonPersistentDocumentFlags.None; string deleteAttachmentChangeVector = null; using (GetConflictsIdPrefix(context, lowerId, out Slice prefixSlice)) { var conflictsTable = context.Transaction.InnerTransaction.OpenTable(ConflictsSchema, ConflictsSlice); conflictsTable.DeleteForwardFrom(ConflictsSchema.Indexes[IdAndChangeVectorSlice], prefixSlice, true, long.MaxValue, conflictDocument => { var conflicted = TableValueToConflictDocument(context, ref conflictDocument.Reader); var collection = _documentsStorage.ExtractCollectionName(context, conflicted.Collection); if (conflicted.Doc != null) { if (conflicted.Flags.Contain(DocumentFlags.HasCounters)) { nonPersistentFlags |= NonPersistentDocumentFlags.ResolveCountersConflict; } _documentsStorage.RevisionsStorage.Put( context, conflicted.Id, conflicted.Doc, conflicted.Flags | DocumentFlags.Conflicted | DocumentFlags.HasRevisions, nonPersistentFlags, conflicted.ChangeVector, conflicted.LastModified.Ticks, collectionName: collection, configuration: RevisionsStorage.ConflictConfiguration.Default); } else if (conflicted.Flags.Contain(DocumentFlags.FromReplication) == false) { using (Slice.External(context.Allocator, conflicted.LowerId, out var key)) { var lastModifiedTicks = _documentDatabase.Time.GetUtcNow().Ticks; _documentsStorage.RevisionsStorage.DeleteRevision(context, key, conflicted.Collection, conflicted.ChangeVector, lastModifiedTicks); } } _documentsStorage.EnsureLastEtagIsPersisted(context, conflicted.Etag); changeVectors.Add(conflicted.ChangeVector); if (conflicted.Flags.Contain(DocumentFlags.HasAttachments) == false) { return; } if (string.IsNullOrEmpty(deleteAttachmentChangeVector)) { var newEtag = _documentsStorage.GenerateNextEtag(); var currentChangeVector = context.LastDatabaseChangeVector ?? GetDatabaseChangeVector(context); deleteAttachmentChangeVector = ChangeVectorUtils .TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentDatabase.DbBase64Id, newEtag, currentChangeVector).ChangeVector; context.LastDatabaseChangeVector = deleteAttachmentChangeVector; } nonPersistentFlags |= DeleteAttachmentConflicts(context, lowerId, document, conflictDocument, deleteAttachmentChangeVector); }); } // once this value has been set, we can't set it to false // an older transaction may be running and seeing it is false it // will not detect a conflict. It is an optimization only that // we have to do, so we'll handle it. var listCount = changeVectors.Count; if (listCount == 0) // there were no conflicts for this document { return(changeVectors, nonPersistentFlags); } // Only register the event if we actually deleted any conflicts var tx = context.Transaction.InnerTransaction.LowLevelTransaction; tx.AfterCommitWhenNewReadTransactionsPrevented += () => { Interlocked.Add(ref ConflictsCount, -listCount); }; return(changeVectors, nonPersistentFlags | NonPersistentDocumentFlags.Resolved); }
public override int Execute(DocumentsOperationContext context) { if (Database.ServerStore.Configuration.Core.FeaturesAvailability == FeaturesAvailability.Stable) { FeaturesAvailabilityException.Throw("Cluster Transactions"); } var global = context.LastDatabaseChangeVector ?? (context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context)); var dbGrpId = Database.DatabaseGroupId; var current = ChangeVectorUtils.GetEtagById(global, dbGrpId); foreach (var command in _batch) { Replies.Add(command.Index, new DynamicJsonArray()); Reply = Replies[command.Index]; var commands = command.Commands; var count = command.PreviousCount; var options = Options[command.Index] = command.Options; if (options.WaitForIndexesTimeout != null) { ModifiedCollections = new HashSet <string>(); } if (commands != null) { foreach (BlittableJsonReaderObject blittableCommand in commands) { count++; var changeVector = $"RAFT:{count}-{dbGrpId}"; var cmd = JsonDeserializationServer.ClusterTransactionDataCommand(blittableCommand); switch (cmd.Type) { case CommandType.PUT: if (current < count) { // delete the document to avoid exception if we put new document in a different collection. // TODO: document this behavior using (DocumentIdWorker.GetSliceFromId(context, cmd.Id, out Slice lowerId)) { Database.DocumentsStorage.Delete(context, lowerId, cmd.Id, expectedChangeVector: null, nonPersistentFlags: NonPersistentDocumentFlags.SkipRevisionCreation); } var putResult = Database.DocumentsStorage.Put(context, cmd.Id, null, cmd.Document.Clone(context), changeVector: changeVector, flags: DocumentFlags.FromClusterTransaction); context.DocumentDatabase.HugeDocuments.AddIfDocIsHuge(cmd.Id, cmd.Document.Size); AddPutResult(putResult); } else { try { var item = Database.DocumentsStorage.GetDocumentOrTombstone(context, cmd.Id); if (item.Missing) { AddPutResult(new DocumentsStorage.PutOperationResults { ChangeVector = changeVector, Id = cmd.Id, LastModified = DateTime.UtcNow, Collection = Database.DocumentsStorage.ExtractCollectionName(context, cmd.Document) }); continue; } var collection = GetCollection(context, item); AddPutResult(new DocumentsStorage.PutOperationResults { ChangeVector = changeVector, Id = cmd.Id, Flags = item.Document?.Flags ?? item.Tombstone.Flags, LastModified = item.Document?.LastModified ?? item.Tombstone.LastModified, Collection = collection }); } catch (DocumentConflictException) { AddPutResult(new DocumentsStorage.PutOperationResults { ChangeVector = changeVector, Id = cmd.Id, Collection = GetFirstConflictCollection(context, cmd) }); } } break; case CommandType.DELETE: if (current < count) { using (DocumentIdWorker.GetSliceFromId(context, cmd.Id, out Slice lowerId)) { var deleteResult = Database.DocumentsStorage.Delete(context, lowerId, cmd.Id, null, changeVector: changeVector, documentFlags: DocumentFlags.FromClusterTransaction); AddDeleteResult(deleteResult, cmd.Id); } } else { try { var item = Database.DocumentsStorage.GetDocumentOrTombstone(context, cmd.Id); if (item.Missing) { AddDeleteResult(new DocumentsStorage.DeleteOperationResult { ChangeVector = changeVector, Collection = null }, cmd.Id); continue; } var collection = GetCollection(context, item); AddDeleteResult(new DocumentsStorage.DeleteOperationResult { ChangeVector = changeVector, Collection = collection }, cmd.Id); } catch (DocumentConflictException) { AddDeleteResult(new DocumentsStorage.DeleteOperationResult { ChangeVector = changeVector, Collection = GetFirstConflictCollection(context, cmd) }, cmd.Id); } } break; default: throw new NotSupportedException($"{cmd.Type} is not supported in {nameof(ClusterTransactionMergedCommand)}."); } } } if (context.LastDatabaseChangeVector == null) { context.LastDatabaseChangeVector = global; } var result = ChangeVectorUtils.TryUpdateChangeVector("RAFT", dbGrpId, count, context.LastDatabaseChangeVector); if (result.IsValid) { context.LastDatabaseChangeVector = result.ChangeVector; } } return(Reply.Count); }
public void EtagShouldNotOverflow2() { var x = ChangeVectorUtils.TryUpdateChangeVector("C", "n0rGjcmUT0u7ctxBXlZZPg", 5554138256, "C:5554138256-n0rGjcmUT0u7ctxBXlZZPg"); Assert.False(x.IsValid); }