private CollectionName GetCollectionFor(DocumentsOperationContext context, Slice prefixSlice)
{
var table = new Table(RevisionsSchema, context.Transaction.InnerTransaction);
var tvr = table.SeekOneForwardFrom(RevisionsSchema.Indexes[IdAndEtagSlice], prefixSlice);
if (tvr == null)
{
return(null);
}
var ptr = tvr.Reader.Read((int)RevisionsTable.Document, out int size);
var data = new BlittableJsonReaderObject(ptr, size, context);
return(_documentsStorage.ExtractCollectionName(context, data));
}
public PutOperationResults PutDocument(DocumentsOperationContext context, string id,
string expectedChangeVector,
BlittableJsonReaderObject document,
long?lastModifiedTicks = null,
string changeVector = null,
DocumentFlags flags = DocumentFlags.None,
NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None)
{
if (context.Transaction == null)
{
ThrowRequiresTransaction();
return(default(PutOperationResults)); // never hit
}
#if DEBUG
var documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
#endif
var newEtag = _documentsStorage.GenerateNextEtag();
var modifiedTicks = lastModifiedTicks ?? _documentDatabase.Time.GetUtcNow().Ticks;
id = BuildDocumentId(id, newEtag, out bool knownNewId);
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
var collectionName = _documentsStorage.ExtractCollectionName(context, document);
var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));
var oldValue = default(TableValueReader);
if (knownNewId == false)
{
// delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it
DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size);
table.ReadByKey(lowerId, out oldValue);
}
BlittableJsonReaderObject oldDoc = null;
if (oldValue.Pointer == null)
{
// expectedChangeVector being null means we don't care, and empty
// means that it must be new
if (string.IsNullOrEmpty(expectedChangeVector) == false)
{
ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector);
}
}
else
{
// expectedChangeVector has special meaning here
// null - means, don't care, don't check
// "" / empty - means, must be new
// anything else - must match exactly
if (expectedChangeVector != null)
{
var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue);
if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0)
{
ThrowConcurrentException(id, expectedChangeVector, oldChangeVector);
}
}
oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context);
var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc);
if (oldCollectionName != collectionName)
{
ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName);
}
var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue);
if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate &&
(nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication)
{
if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
{
flags |= DocumentFlags.HasAttachments;
}
}
}
var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue);
changeVector = result.ChangeVector;
nonPersistentFlags |= result.NonPersistentFlags;
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved))
{
flags |= DocumentFlags.Resolved;
}
if (collectionName.IsHiLo == false &&
(flags & DocumentFlags.Artificial) != DocumentFlags.Artificial)
{
if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags))
{
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
#endif
document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
#if DEBUG
documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
}
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false &&
(flags.Contain(DocumentFlags.Resolved) ||
_documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null
))
{
var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document,
ref flags, out RevisionsCollectionConfiguration configuration);
if (shouldVersion)
{
_documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags,
changeVector, modifiedTicks, configuration, collectionName);
}
}
}
using (Slice.From(context.Allocator, changeVector, out var cv))
using (table.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(Bits.SwapBytes(newEtag));
tvb.Add(idPtr);
tvb.Add(document.BasePointer, document.Size);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(modifiedTicks);
tvb.Add((int)flags);
tvb.Add(context.GetTransactionMarker());
if (oldValue.Pointer == null)
{
table.Insert(tvb);
}
else
{
table.Update(oldValue.Id, tvb);
}
}
if (collectionName.IsHiLo == false)
{
_documentsStorage.ExpirationStorage.Put(context, lowerId, document);
}
context.LastDatabaseChangeVector = changeVector;
_documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1);
_documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size);
context.Transaction.AddAfterCommitNotification(new DocumentChange
{
ChangeVector = changeVector,
CollectionName = collectionName.Name,
Id = id,
Type = DocumentChangeTypes.Put,
});
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
return(new PutOperationResults
{
Etag = newEtag,
Id = id,
Collection = collectionName,
ChangeVector = changeVector,
Flags = flags,
LastModified = new DateTime(modifiedTicks)
});
}
}
public (List <string> ChangeVectors, NonPersistentDocumentFlags NonPersistentFlags) DeleteConflictsFor(
DocumentsOperationContext context, Slice lowerId, BlittableJsonReaderObject document)
{
if (ConflictsCount == 0)
{
return(null, NonPersistentDocumentFlags.None);
}
var changeVectors = new List <string>();
var nonPersistentFlags = NonPersistentDocumentFlags.None;
string deleteAttachmentChangeVector = null;
using (GetConflictsIdPrefix(context, lowerId, out Slice prefixSlice))
{
var conflictsTable = context.Transaction.InnerTransaction.OpenTable(ConflictsSchema, ConflictsSlice);
conflictsTable.DeleteForwardFrom(ConflictsSchema.Indexes[IdAndChangeVectorSlice], prefixSlice, true, long.MaxValue, conflictDocument =>
{
var conflicted = TableValueToConflictDocument(context, ref conflictDocument.Reader);
var collection = _documentsStorage.ExtractCollectionName(context, conflicted.Collection);
if (conflicted.Doc != null)
{
_documentsStorage.RevisionsStorage.Put(
context, conflicted.Id, conflicted.Doc, conflicted.Flags | DocumentFlags.Conflicted, nonPersistentFlags, conflicted.ChangeVector,
conflicted.LastModified.Ticks,
collectionName: collection, configuration: RevisionsStorage.ConflictConfiguration.Default);
}
else if (conflicted.Flags.Contain(DocumentFlags.FromReplication) == false)
{
using (Slice.External(context.Allocator, conflicted.LowerId, out var key))
{
var lastModifiedTicks = _documentDatabase.Time.GetUtcNow().Ticks;
_documentsStorage.RevisionsStorage.DeleteRevision(context, key, conflicted.Collection, conflicted.ChangeVector, lastModifiedTicks);
}
}
_documentsStorage.EnsureLastEtagIsPersisted(context, conflicted.Etag);
changeVectors.Add(conflicted.ChangeVector);
if (conflicted.Flags.Contain(DocumentFlags.HasAttachments) == false)
{
return;
}
if (string.IsNullOrEmpty(deleteAttachmentChangeVector))
{
var newEtag = _documentsStorage.GenerateNextEtag();
deleteAttachmentChangeVector = _documentsStorage.GetNewChangeVector(context, newEtag);
context.LastDatabaseChangeVector = conflicted.ChangeVector;
}
nonPersistentFlags |= DeleteAttachmentConflicts(context, lowerId, document, conflictDocument, deleteAttachmentChangeVector);
});
}
// once this value has been set, we can't set it to false
// an older transaction may be running and seeing it is false it
// will not detect a conflict. It is an optimization only that
// we have to do, so we'll handle it.
// Only register the event if we actually deleted any conflicts
var listCount = changeVectors.Count;
if (listCount > 0)
{
var tx = context.Transaction.InnerTransaction.LowLevelTransaction;
tx.AfterCommitWhenNewReadTransactionsPrevented += () =>
{
Interlocked.Add(ref ConflictsCount, -listCount);
};
}
return(changeVectors, nonPersistentFlags | NonPersistentDocumentFlags.Resolved);
}
private void PutCounterImpl(DocumentsOperationContext context, string documentId, string collection, string name, string changeVector, long value)
{
if (context.Transaction == null)
{
DocumentPutAction.ThrowRequiresTransaction();
Debug.Assert(false);// never hit
}
var collectionName = _documentsStorage.ExtractCollectionName(context, collection);
var table = GetCountersTable(context.Transaction.InnerTransaction, collectionName);
using (GetCounterKey(context, documentId, name, changeVector ?? context.Environment.Base64Id, out var counterKey))
{
using (DocumentIdWorker.GetStringPreserveCase(context, name, out Slice nameSlice))
using (table.Allocate(out TableValueBuilder tvb))
{
if (changeVector != null)
{
if (table.ReadByKey(counterKey, out var existing))
{
var existingChangeVector = TableValueToChangeVector(context, (int)CountersTable.ChangeVector, ref existing);
if (ChangeVectorUtils.GetConflictStatus(changeVector, existingChangeVector) == ConflictStatus.AlreadyMerged)
{
return;
}
}
}
RemoveTombstoneIfExists(context, documentId, name);
var etag = _documentsStorage.GenerateNextEtag();
if (changeVector == null)
{
changeVector = ChangeVectorUtils
.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentsStorage.Environment.Base64Id, etag, string.Empty)
.ChangeVector;
}
using (Slice.From(context.Allocator, changeVector, out var cv))
using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice))
{
tvb.Add(counterKey);
tvb.Add(nameSlice);
tvb.Add(Bits.SwapBytes(etag));
tvb.Add(value);
tvb.Add(cv);
tvb.Add(collectionSlice);
tvb.Add(context.TransactionMarkerOffset);
table.Set(tvb);
}
UpdateMetrics(counterKey, name, changeVector, collection);
context.Transaction.AddAfterCommitNotification(new CounterChange
{
ChangeVector = changeVector,
DocumentId = documentId,
Name = name,
Value = value,
Type = CounterChangeTypes.Put
});
}
}
}
public void AddConflict(
DocumentsOperationContext context,
string id,
long lastModifiedTicks,
BlittableJsonReaderObject incomingDoc,
string incomingChangeVector,
string incomingTombstoneCollection,
DocumentFlags flags)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Adding conflict to {id} (Incoming change vector {incomingChangeVector})");
}
var tx = context.Transaction.InnerTransaction;
var conflictsTable = tx.OpenTable(ConflictsSchema, ConflictsSlice);
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
CollectionName collectionName;
// ReSharper disable once ArgumentsStyleLiteral
var existing = _documentsStorage.GetDocumentOrTombstone(context, id, throwOnConflict: false);
LazyStringValue lazyCollectionName;
if (existing.Document != null)
{
var existingDoc = existing.Document;
lazyCollectionName = CollectionName.GetLazyCollectionNameFrom(context, existingDoc.Data);
using (Slice.From(context.Allocator, existingDoc.ChangeVector, out var cv))
using (conflictsTable.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(idPtr);
tvb.Add(existingDoc.Data.BasePointer, existingDoc.Data.Size);
tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag()));
tvb.Add(lazyCollectionName.Buffer, lazyCollectionName.Size);
tvb.Add(existingDoc.LastModified.Ticks);
tvb.Add((int)existingDoc.Flags);
if (conflictsTable.Set(tvb))
{
Interlocked.Increment(ref ConflictsCount);
}
}
// we delete the data directly, without generating a tombstone, because we have a
// conflict instead
_documentsStorage.EnsureLastEtagIsPersisted(context, existingDoc.Etag);
collectionName = _documentsStorage.ExtractCollectionName(context, existingDoc.Id, existingDoc.Data);
//make sure that the relevant collection tree exists
var table = tx.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));
table.Delete(existingDoc.StorageId);
}
else if (existing.Tombstone != null)
{
var existingTombstone = existing.Tombstone;
using (Slice.From(context.Allocator, existingTombstone.ChangeVector, out var cv))
using (conflictsTable.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(idPtr);
tvb.Add(null, 0);
tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag()));
tvb.Add(existingTombstone.Collection.Buffer, existingTombstone.Collection.Size);
tvb.Add(existingTombstone.LastModified.Ticks);
tvb.Add((int)existingTombstone.Flags);
if (conflictsTable.Set(tvb))
{
Interlocked.Increment(ref ConflictsCount);
}
}
// we delete the data directly, without generating a tombstone, because we have a
// conflict instead
_documentsStorage.EnsureLastEtagIsPersisted(context, existingTombstone.Etag);
collectionName = _documentsStorage.GetCollection(existingTombstone.Collection, throwIfDoesNotExist: true);
var table = tx.OpenTable(TombstonesSchema, collectionName.GetTableName(CollectionTableType.Tombstones));
table.Delete(existingTombstone.StorageId);
}
else // has existing conflicts
{
collectionName = _documentsStorage.ExtractCollectionName(context, id, incomingDoc);
using (GetConflictsIdPrefix(context, lowerId, out Slice prefixSlice))
{
var conflicts = GetConflictsFor(context, prefixSlice);
foreach (var conflict in conflicts)
{
var conflictStatus = ChangeVectorUtils.GetConflictStatus(incomingChangeVector, conflict.ChangeVector);
switch (conflictStatus)
{
case ConflictStatus.Update:
DeleteConflictsFor(context, conflict.ChangeVector); // delete this, it has been subsumed
break;
case ConflictStatus.Conflict:
break; // we'll add this conflict if no one else also includes it
case ConflictStatus.AlreadyMerged:
return; // we already have a conflict that includes this version
default:
throw new ArgumentOutOfRangeException("Invalid conflict status " + conflictStatus);
}
}
}
}
var etag = _documentsStorage.GenerateNextEtag();
ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentDatabase.DbId, etag, ref context.LastDatabaseChangeVector);
byte *doc = null;
var docSize = 0;
if (incomingDoc != null) // can be null if it is a tombstone
{
doc = incomingDoc.BasePointer;
docSize = incomingDoc.Size;
lazyCollectionName = CollectionName.GetLazyCollectionNameFrom(context, incomingDoc);
}
else
{
lazyCollectionName = context.GetLazyString(incomingTombstoneCollection);
}
using (lazyCollectionName)
using (Slice.From(context.Allocator, incomingChangeVector, out var cv))
using (conflictsTable.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(idPtr);
tvb.Add(doc, docSize);
tvb.Add(Bits.SwapBytes(etag));
tvb.Add(lazyCollectionName.Buffer, lazyCollectionName.Size);
tvb.Add(lastModifiedTicks);
tvb.Add((int)flags);
if (conflictsTable.Set(tvb))
{
Interlocked.Increment(ref ConflictsCount);
}
}
context.Transaction.AddAfterCommitNotification(new DocumentChange
{
ChangeVector = incomingChangeVector,
CollectionName = collectionName.Name,
Id = id,
Type = DocumentChangeTypes.Conflict,
IsSystemDocument = false
});
}
}
