public void Delete(DocumentsOperationContext context, string id, BlittableJsonReaderObject deleteRevisionDocument,
DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags, string changeVector, long lastModifiedTicks)
{
BlittableJsonReaderObject.AssertNoModifications(deleteRevisionDocument, id, assertChildren: true);
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
var collectionName = _documentsStorage.ExtractCollectionName(context, deleteRevisionDocument);
Delete(context, lowerId, idPtr, id, collectionName, deleteRevisionDocument, changeVector, lastModifiedTicks, nonPersistentFlags, flags);
}
}
public static unsafe DocumentCompareResult IsEqualTo(BlittableJsonReaderObject original, BlittableJsonReaderObject modified,
bool tryMergeAttachmentsConflict)
{
if (ReferenceEquals(original, modified))
{
return(DocumentCompareResult.Equal);
}
if (original == null || modified == null)
{
return(DocumentCompareResult.NotEqual);
}
BlittableJsonReaderObject.AssertNoModifications(original, nameof(original), true);
BlittableJsonReaderObject.AssertNoModifications(modified, nameof(modified), true);
if (original.Size == modified.Size)
{
// if this didn't change, we can check the raw memory directly.
if (Memory.Compare(original.BasePointer, modified.BasePointer, original.Size) == 0)
{
return(DocumentCompareResult.Equal);
}
}
// Performance improvemnt: We compare the metadata first
// because that most of the time the metadata itself won't be the equal, so no need to compare all values
var result = IsMetadataEqualTo(original, modified, tryMergeAttachmentsConflict);
if (result == DocumentCompareResult.NotEqual)
{
return(DocumentCompareResult.NotEqual);
}
if (ComparePropertiesExceptStartingWithAt(original, modified) == DocumentCompareResult.NotEqual)
{
return(DocumentCompareResult.NotEqual);
}
return(result);
}
public PutOperationResults PutDocument(DocumentsOperationContext context, string id,
string expectedChangeVector,
BlittableJsonReaderObject document,
long?lastModifiedTicks = null,
string changeVector = null,
DocumentFlags flags = DocumentFlags.None,
NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None)
{
if (context.Transaction == null)
{
ThrowRequiresTransaction();
return(default(PutOperationResults)); // never hit
}
#if DEBUG
var documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
#endif
var newEtag = _documentsStorage.GenerateNextEtag();
var modifiedTicks = lastModifiedTicks ?? _documentDatabase.Time.GetUtcNow().Ticks;
id = BuildDocumentId(id, newEtag, out bool knownNewId);
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
var collectionName = _documentsStorage.ExtractCollectionName(context, document);
var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));
var oldValue = default(TableValueReader);
if (knownNewId == false)
{
// delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it
DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size);
table.ReadByKey(lowerId, out oldValue);
}
BlittableJsonReaderObject oldDoc = null;
if (oldValue.Pointer == null)
{
// expectedChangeVector being null means we don't care, and empty
// means that it must be new
if (string.IsNullOrEmpty(expectedChangeVector) == false)
{
ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector);
}
}
else
{
// expectedChangeVector has special meaning here
// null - means, don't care, don't check
// "" / empty - means, must be new
// anything else - must match exactly
if (expectedChangeVector != null)
{
var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue);
if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0)
{
ThrowConcurrentException(id, expectedChangeVector, oldChangeVector);
}
}
oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context);
var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc);
if (oldCollectionName != collectionName)
{
ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName);
}
var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue);
if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate &&
(nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication)
{
if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
{
flags |= DocumentFlags.HasAttachments;
}
}
}
var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue);
changeVector = result.ChangeVector;
nonPersistentFlags |= result.NonPersistentFlags;
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved))
{
flags |= DocumentFlags.Resolved;
}
if (collectionName.IsHiLo == false &&
(flags & DocumentFlags.Artificial) != DocumentFlags.Artificial)
{
if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags))
{
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
#endif
document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
#if DEBUG
documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
}
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false &&
(flags.Contain(DocumentFlags.Resolved) ||
_documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null
))
{
var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document,
ref flags, out RevisionsCollectionConfiguration configuration);
if (shouldVersion)
{
_documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags,
changeVector, modifiedTicks, configuration, collectionName);
}
}
}
using (Slice.From(context.Allocator, changeVector, out var cv))
using (table.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(Bits.SwapBytes(newEtag));
tvb.Add(idPtr);
tvb.Add(document.BasePointer, document.Size);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(modifiedTicks);
tvb.Add((int)flags);
tvb.Add(context.GetTransactionMarker());
if (oldValue.Pointer == null)
{
table.Insert(tvb);
}
else
{
table.Update(oldValue.Id, tvb);
}
}
if (collectionName.IsHiLo == false)
{
_documentsStorage.ExpirationStorage.Put(context, lowerId, document);
}
context.LastDatabaseChangeVector = changeVector;
_documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1);
_documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size);
context.Transaction.AddAfterCommitNotification(new DocumentChange
{
ChangeVector = changeVector,
CollectionName = collectionName.Name,
Id = id,
Type = DocumentChangeTypes.Put,
});
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
return(new PutOperationResults
{
Etag = newEtag,
Id = id,
Collection = collectionName,
ChangeVector = changeVector,
Flags = flags,
LastModified = new DateTime(modifiedTicks)
});
}
}
public void Put(DocumentsOperationContext context, string id, BlittableJsonReaderObject document,
DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags, string changeVector, long lastModifiedTicks,
RevisionsCollectionConfiguration configuration = null, CollectionName collectionName = null)
{
Debug.Assert(changeVector != null, "Change vector must be set");
Debug.Assert(lastModifiedTicks != DateTime.MinValue.Ticks, "last modified ticks must be set");
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
if (collectionName == null)
{
collectionName = _database.DocumentsStorage.ExtractCollectionName(context, document);
}
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
var fromSmuggler = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler;
var fromReplication = (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) == NonPersistentDocumentFlags.FromReplication;
var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName);
// We want the revision's attachments to have a lower etag than the revision itself
if ((flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments &&
fromSmuggler == false)
{
using (Slice.From(context.Allocator, changeVector, out Slice changeVectorSlice))
{
if (table.VerifyKeyExists(changeVectorSlice) == false)
{
_documentsStorage.AttachmentsStorage.RevisionAttachments(context, lowerId, changeVectorSlice);
}
}
}
if (fromReplication)
{
void PutFromRevisionIfChangeVectorIsGreater()
{
bool hasDoc;
TableValueReader tvr;
try
{
hasDoc = _documentsStorage.GetTableValueReaderForDocument(context, lowerId, throwOnConflict: true, tvr: out tvr);
}
catch (DocumentConflictException)
{
// Do not modify the document.
return;
}
if (hasDoc == false)
{
PutFromRevision();
return;
}
var docChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref tvr);
if (ChangeVectorUtils.GetConflictStatus(changeVector, docChangeVector) == ConflictStatus.Update)
{
PutFromRevision();
}
void PutFromRevision()
{
_documentsStorage.Put(context, id, null, document, lastModifiedTicks, changeVector,
flags & ~DocumentFlags.Revision, nonPersistentFlags | NonPersistentDocumentFlags.FromRevision);
}
}
PutFromRevisionIfChangeVectorIsGreater();
}
flags |= DocumentFlags.Revision;
var newEtag = _database.DocumentsStorage.GenerateNextEtag();
var newEtagSwapBytes = Bits.SwapBytes(newEtag);
using (table.Allocate(out TableValueBuilder tvb))
using (Slice.From(context.Allocator, changeVector, out var cv))
{
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(newEtagSwapBytes);
tvb.Add(idPtr);
tvb.Add(document.BasePointer, document.Size);
tvb.Add((int)flags);
tvb.Add(NotDeletedRevisionMarker);
tvb.Add(lastModifiedTicks);
tvb.Add(context.GetTransactionMarker());
if (flags.Contain(DocumentFlags.Resolved))
{
tvb.Add((int)DocumentFlags.Resolved);
}
else
{
tvb.Add(0);
}
tvb.Add(Bits.SwapBytes(lastModifiedTicks));
var isNew = table.Set(tvb);
if (isNew == false)
{
// It might be just an update from replication as we call this twice, both for the doc delete and for deleteRevision.
return;
}
}
if (configuration == null)
{
configuration = GetRevisionsConfiguration(collectionName.Name);
}
DeleteOldRevisions(context, table, lowerId, collectionName, configuration, nonPersistentFlags, changeVector, lastModifiedTicks);
}
}
private static unsafe bool CompareBlittable(string fieldPath, string id, BlittableJsonReaderObject originalBlittable,
BlittableJsonReaderObject newBlittable, IDictionary <string, DocumentsChanges[]> changes,
List <DocumentsChanges> docChanges)
{
BlittableJsonReaderObject.AssertNoModifications(originalBlittable, id, assertChildren: false);
BlittableJsonReaderObject.AssertNoModifications(newBlittable, id, assertChildren: false);
var newBlittableProps = newBlittable.GetPropertyNames();
var oldBlittableProps = originalBlittable.GetPropertyNames();
var newFields = new HashSet <string>(newBlittableProps);
newFields.ExceptWith(oldBlittableProps);
var removedFields = new HashSet <string>(oldBlittableProps);
removedFields.ExceptWith(newBlittableProps);
using var orderedProperties = newBlittable.GetPropertiesByInsertionOrder();
foreach (var field in removedFields)
{
if (changes == null)
{
return(true);
}
if (field.Equals(LastModified) ||
field.Equals(ChangeVector) ||
field.Equals(Id))
{
continue;
}
NewChange(fieldPath, field, null, null, docChanges, DocumentsChanges.ChangeType.RemovedField);
}
var newProp = new BlittableJsonReaderObject.PropertyDetails();
var oldProp = new BlittableJsonReaderObject.PropertyDetails();
for (int i = 0; i < orderedProperties.Size; i++)
{
newBlittable.GetPropertyByIndex(orderedProperties.Properties[i], ref newProp);
if (newProp.Name.Equals(LastModified) ||
newProp.Name.Equals(Collection) ||
newProp.Name.Equals(ChangeVector) ||
newProp.Name.Equals(Id))
{
continue;
}
if (newFields.Contains(newProp.Name))
{
if (changes == null)
{
return(true);
}
NewChange(fieldPath, newProp.Name, newProp.Value, null, docChanges, DocumentsChanges.ChangeType.NewField);
continue;
}
var oldPropId = originalBlittable.GetPropertyIndex(newProp.Name);
originalBlittable.GetPropertyByIndex(oldPropId, ref oldProp);
switch ((newProp.Token & BlittableJsonReaderBase.TypesMask))
{
case BlittableJsonToken.Integer:
case BlittableJsonToken.Boolean:
case BlittableJsonToken.LazyNumber:
case BlittableJsonToken.CompressedString:
case BlittableJsonToken.String:
if (newProp.Value.Equals(oldProp.Value) || CompareValues(oldProp, newProp) ||
CompareStringsWithEscapePositions(newBlittable._context, oldProp, newProp))
{
break;
}
if (changes == null)
{
return(true);
}
NewChange(fieldPath, newProp.Name, newProp.Value, oldProp.Value, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
break;
case BlittableJsonToken.Null:
if (oldProp.Value == null)
{
break;
}
if (changes == null)
{
return(true);
}
NewChange(fieldPath, newProp.Name, null, oldProp.Value, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
break;
case BlittableJsonToken.StartArray:
var newArray = newProp.Value as BlittableJsonReaderArray;
var oldArray = oldProp.Value as BlittableJsonReaderArray;
if (newArray == null)
{
throw new InvalidDataException($"Invalid blittable, expected array but got {newProp.Value}");
}
if (oldArray == null)
{
if (changes == null)
{
return(true);
}
NewChange(fieldPath, newProp.Name, newProp.Value, oldProp.Value, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
break;
}
var changed = CompareBlittableArray(FieldPathCombine(fieldPath, newProp.Name), id, oldArray, newArray, changes, docChanges, newProp.Name);
if (changes == null && changed)
{
return(true);
}
break;
case BlittableJsonToken.StartObject:
if (oldProp.Value == null ||
!(oldProp.Value is BlittableJsonReaderObject oldObj))
{
if (changes == null)
{
return(true);
}
NewChange(fieldPath, newProp.Name, newProp.Value, oldProp.Value, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
break;
}
if (!(newProp.Value is BlittableJsonReaderObject newObj))
{
throw new InvalidDataException($"Invalid blittable, expected object but got {newProp.Value}");
}
changed = CompareBlittable(FieldPathCombine(fieldPath, newProp.Name), id, oldObj, newObj, changes, docChanges);
if (changes == null && changed)
{
return(true);
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
if ((changes == null) || (docChanges.Count <= 0))
{
return(false);
}
changes[id] = docChanges.ToArray();
return(true);
}
private static bool CompareBlittable(string id, BlittableJsonReaderObject originalBlittable,
BlittableJsonReaderObject newBlittable, IDictionary <string, DocumentsChanges[]> changes,
List <DocumentsChanges> docChanges)
{
BlittableJsonReaderObject.AssertNoModifications(originalBlittable, id, assertChildren: false);
BlittableJsonReaderObject.AssertNoModifications(newBlittable, id, assertChildren: false);
var newBlittableProps = newBlittable.GetPropertyNames();
var oldBlittableProps = originalBlittable.GetPropertyNames();
var newFields = newBlittableProps.Except(oldBlittableProps);
var removedFields = oldBlittableProps.Except(newBlittableProps);
var propertiesIds = newBlittable.GetPropertiesByInsertionOrder();
foreach (var field in removedFields)
{
if (changes == null)
{
return(true);
}
NewChange(field, null, null, docChanges, DocumentsChanges.ChangeType.RemovedField);
}
var newProp = new BlittableJsonReaderObject.PropertyDetails();
var oldProp = new BlittableJsonReaderObject.PropertyDetails();
foreach (var propId in propertiesIds)
{
newBlittable.GetPropertyByIndex(propId, ref newProp);
if (newProp.Name.Equals(LastModified) ||
newProp.Name.Equals(Collection) ||
newProp.Name.Equals(ChangeVector) ||
newProp.Name.Equals(Id))
{
continue;
}
if (newFields.Contains(newProp.Name))
{
if (changes == null)
{
return(true);
}
NewChange(newProp.Name, newProp.Value, null, docChanges, DocumentsChanges.ChangeType.NewField);
continue;
}
var oldPropId = originalBlittable.GetPropertyIndex(newProp.Name);
originalBlittable.GetPropertyByIndex(oldPropId, ref oldProp);
switch ((newProp.Token & BlittableJsonReaderBase.TypesMask))
{
case BlittableJsonToken.Integer:
case BlittableJsonToken.Boolean:
case BlittableJsonToken.LazyNumber:
case BlittableJsonToken.CompressedString:
case BlittableJsonToken.String:
if (newProp.Value.Equals(oldProp.Value) || ComapreValues(oldProp, newProp))
{
break;
}
if (changes == null)
{
return(true);
}
NewChange(newProp.Name, newProp.Value, oldProp.Value, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
break;
case BlittableJsonToken.Null:
if (oldProp.Value == null)
{
break;
}
if (changes == null)
{
return(true);
}
NewChange(newProp.Name, null, oldProp.Value, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
break;
case BlittableJsonToken.StartArray:
var newArray = newProp.Value as BlittableJsonReaderArray;
var oldArray = oldProp.Value as BlittableJsonReaderArray;
if ((newArray == null) || (oldArray == null))
{
throw new InvalidDataException("Invalid blittable");
}
var changed = CompareBlittableArray(id, oldArray, newArray, changes, docChanges, newProp.Name);
if (changed == false)
{
break;
}
if (changes == null)
{
return(true);
}
break;
case BlittableJsonToken.StartObject:
if (oldProp.Value == null)
{
if (changes == null)
{
return(true);
}
changed = true;
NewChange(newProp.Name, newProp.Value, null, docChanges,
DocumentsChanges.ChangeType.FieldChanged);
}
else
{
changed = CompareBlittable(id, oldProp.Value as BlittableJsonReaderObject,
newProp.Value as BlittableJsonReaderObject, changes, docChanges);
}
if ((changes == null) && (changed))
{
return(true);
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
if ((changes == null) || (docChanges.Count <= 0))
{
return(false);
}
changes[id] = docChanges.ToArray();
return(true);
}