private bool TryResolveConflictByScript(
DocumentsOperationContext documentsContext,
DocumentConflict conflict)
{
var collection = conflict.Collection;
var hasScript = _conflictResolver.ScriptConflictResolversCache.TryGetValue(collection, out ScriptResolver scriptResolver);
if (!hasScript || scriptResolver == null)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Script not found to resolve the {collection} collection");
}
return(false);
}
var conflictedDocs = new List <DocumentConflict>(documentsContext.DocumentDatabase.DocumentsStorage.ConflictsStorage.GetConflictsFor(documentsContext, conflict.Id));
if (conflictedDocs.Count == 0)
{
var relevantLocalDoc = documentsContext.DocumentDatabase.DocumentsStorage
.GetDocumentOrTombstone(
documentsContext,
conflict.Id);
if (relevantLocalDoc.Document != null)
{
conflictedDocs.Add(DocumentConflict.From(documentsContext, relevantLocalDoc.Document));
}
else if (relevantLocalDoc.Tombstone != null)
{
conflictedDocs.Add(DocumentConflict.From(relevantLocalDoc.Tombstone));
}
}
if (conflictedDocs.Count == 0)
{
InvalidConflictWhenThereIsNone(conflict.Id);
}
conflictedDocs.Add(conflict.Clone());
if (_conflictResolver.TryResolveConflictByScriptInternal(
documentsContext,
scriptResolver,
conflictedDocs,
documentsContext.GetLazyString(collection), out var resolved))
{
_conflictResolver.PutResolvedDocument(documentsContext, resolved, conflict);
return(true);
}
return(false);
}
public JsValue PutDocument(JsValue self, JsValue[] args)
{
string changeVector = null;
if (args.Length != 2 && args.Length != 3)
{
throw new InvalidOperationException("put(id, doc, changeVector) must be called with called with 2 or 3 arguments only");
}
AssertValidDatabaseContext();
AssertNotReadOnly();
if (args[0].IsString() == false && args[0].IsNull() == false && args[0].IsUndefined() == false)
{
AssertValidId();
}
var id = args[0].IsNull() || args[0].IsUndefined() ? null : args[0].AsString();
if (args[1].IsObject() == false)
{
throw new InvalidOperationException(
$"Created document must be a valid object which is not null or empty. Document ID: '{id}'.");
}
PutOrDeleteCalled = true;
if (args.Length == 3)
{
if (args[2].IsString())
{
changeVector = args[2].AsString();
}
else if (args[2].IsNull() == false && args[0].IsUndefined() == false)
{
throw new InvalidOperationException(
$"The change vector must be a string or null. Document ID: '{id}'.");
}
}
if (DebugMode)
{
DebugActions.PutDocument.Add(id);
}
using (var reader = JsBlittableBridge.Translate(_context, ScriptEngine, args[1].AsObject(), usageMode: BlittableJsonDocumentBuilder.UsageMode.ToDisk))
{
var put = _database.DocumentsStorage.Put(_context, id, _context.GetLazyString(changeVector), reader);
return(put.Id);
}
}
protected unsafe void SetLazyStringValueFromString(DocumentsOperationContext context, out LazyStringValue prop)
{
prop = null;
var size = *(int *)Reader.ReadExactly(sizeof(int));
if (size < 0)
{
return;
}
// This is a special (broken) case.
// On the source it is stored as Slice in LSV format which is wrong(?) unlike the normal LSV the escaping position is kept before the value itself.
// and therefore the escaping doesn't include in the LSV size.
// Additionally, we are over-allocating so writing this value doesn't cause a failure (we look for the escaping after the value)
// this also work, because we don't pass those values between contexts, if we need to do so, we convert it to string first.
// TODO: this is inefficient, can skip string allocation
prop = context.GetLazyString(Encoding.UTF8.GetString(Reader.ReadExactly(size), size));
}
private bool TryResolveUsingDefaultResolver(
DocumentsOperationContext context,
string id,
string collection,
string incomingChangeVector,
BlittableJsonReaderObject doc)
{
if (_conflictResolver.ConflictSolver?.DatabaseResolverId == null)
{
return(false);
}
var conflicts = new List <DocumentConflict>(_database.DocumentsStorage.ConflictsStorage.GetConflictsFor(context, id));
var localDocumentTuple = _database.DocumentsStorage.GetDocumentOrTombstone(context, id, false);
var localDoc = DocumentConflict.From(context, localDocumentTuple.Document) ??
DocumentConflict.From(localDocumentTuple.Tombstone);
if (localDoc != null)
{
conflicts.Add(localDoc);
}
conflicts.Add(new DocumentConflict
{
ChangeVector = incomingChangeVector,
Collection = context.GetLazyStringForFieldWithCaching(
collection ??
CollectionName.GetCollectionName(id, doc)),
Doc = doc,
LowerId = context.GetLazyString(id)
});
if (_conflictResolver.TryResolveUsingDefaultResolverInternal(
context,
_conflictResolver.ConflictSolver.DatabaseResolverId,
conflicts, out var resolved))
{
_conflictResolver.PutResolvedDocument(context, resolved);
return(true);
}
return(false);
}
private CollectionName ResolveConflictAndAddTombstone(DocumentsOperationContext context, string changeVector,
IReadOnlyList <DocumentConflict> conflicts, out long etag)
{
var indexOfLargestEtag = FindIndexOfLargestEtagAndMergeChangeVectors(conflicts, out string mergedChangeVector);
var latestConflict = conflicts[indexOfLargestEtag];
var collectionName = new CollectionName(latestConflict.Collection);
using (DocumentIdWorker.GetSliceFromId(context, latestConflict.Id, out Slice lowerId))
{
//note that CreateTombstone is also deleting conflicts
etag = _documentsStorage.CreateTombstone(context,
lowerId,
latestConflict.Etag,
collectionName,
context.GetLazyString(mergedChangeVector),
latestConflict.LastModified.Ticks,
changeVector,
latestConflict.Flags).Etag;
}
return(collectionName);
}
public void HandleConflictForDocument(
DocumentsOperationContext documentsContext,
string id,
string collection,
long lastModifiedTicks,
BlittableJsonReaderObject doc,
string changeVector,
string conflictedChangeVector,
DocumentFlags flags)
{
if (id.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase))
{
HandleHiloConflict(documentsContext, id, doc, changeVector);
return;
}
if (TryResolveIdenticalDocument(
documentsContext,
id,
doc,
lastModifiedTicks,
changeVector))
{
return;
}
var conflictedDoc = new DocumentConflict
{
Doc = doc,
Collection = documentsContext.GetLazyStringForFieldWithCaching(
collection ??
CollectionName.GetCollectionName(doc)
),
LastModified = new DateTime(lastModifiedTicks),
LowerId = documentsContext.GetLazyString(id),
Id = documentsContext.GetLazyString(id),
ChangeVector = changeVector
};
if (TryResolveConflictByScript(
documentsContext,
conflictedDoc))
{
return;
}
if (_conflictResolver.ConflictSolver?.ResolveToLatest == true)
{
if (conflictedChangeVector == null) //precaution
{
throw new InvalidOperationException(
"Detected conflict on replication, but could not figure out conflicted vector. This is not supposed to happen and is likely a bug.");
}
var conflicts = new List <DocumentConflict>
{
conflictedDoc.Clone()
};
conflicts.AddRange(documentsContext.DocumentDatabase.DocumentsStorage.ConflictsStorage.GetConflictsFor(
documentsContext, id));
var localDocumentTuple =
documentsContext.DocumentDatabase.DocumentsStorage.GetDocumentOrTombstone(documentsContext,
id, false);
var local = DocumentConflict.From(documentsContext, localDocumentTuple.Document) ?? DocumentConflict.From(localDocumentTuple.Tombstone);
if (local != null)
{
conflicts.Add(local);
}
var resolved = _conflictResolver.ResolveToLatest(documentsContext, conflicts);
_conflictResolver.PutResolvedDocument(documentsContext, resolved, conflictedDoc);
return;
}
_database.DocumentsStorage.ConflictsStorage.AddConflict(documentsContext, id, lastModifiedTicks, doc, changeVector, collection, flags);
}
private bool TryResolveConflictByScript(
DocumentsOperationContext documentsContext,
string id,
string incomingChangeVector,
BlittableJsonReaderObject doc)
{
var collection = CollectionName.GetCollectionName(id, doc);
var hasScript = _conflictResolver.ScriptConflictResolversCache.TryGetValue(collection, out ScriptResolver scriptResolver);
if (!hasScript || scriptResolver == null)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Script not found to resolve the {collection} collection");
}
return(false);
}
var conflictedDocs = new List <DocumentConflict>(documentsContext.DocumentDatabase.DocumentsStorage.ConflictsStorage.GetConflictsFor(documentsContext, id));
if (conflictedDocs.Count == 0)
{
var relevantLocalDoc = documentsContext.DocumentDatabase.DocumentsStorage
.GetDocumentOrTombstone(
documentsContext,
id);
if (relevantLocalDoc.Document != null)
{
conflictedDocs.Add(DocumentConflict.From(documentsContext, relevantLocalDoc.Document));
}
else if (relevantLocalDoc.Tombstone != null)
{
conflictedDocs.Add(DocumentConflict.From(relevantLocalDoc.Tombstone));
}
}
if (conflictedDocs.Count == 0)
{
InvalidConflictWhenThereIsNone(id);
}
conflictedDocs.Add(new DocumentConflict
{
LowerId = conflictedDocs[0].LowerId,
Id = conflictedDocs[0].Id,
Collection = documentsContext.GetLazyStringForFieldWithCaching(collection),
ChangeVector = incomingChangeVector,
Doc = doc
});
if (_conflictResolver.TryResolveConflictByScriptInternal(
documentsContext,
scriptResolver,
conflictedDocs,
documentsContext.GetLazyString(collection), out var resolved))
{
_conflictResolver.PutResolvedDocument(documentsContext, resolved);
return(true);
}
return(false);
}
private DocumentItem ConvertRecordToDocumentItem(DocumentsOperationContext context, string[] csvReaderCurrentRecord, string[] csvReaderFieldHeaders, string collection)
{
try
{
var idStr = _hasId ? csvReaderCurrentRecord[_idIndex] : _hasCollection ? $"{csvReaderCurrentRecord[_collectionIndex]}/" : $"{collection}/";
var data = new DynamicJsonValue();
for (int i = 0; i < csvReaderFieldHeaders.Length; i++)
{
//ignoring reserved properties
if (csvReaderFieldHeaders[i][0] == '@')
{
if (_hasCollection && i == _collectionIndex)
{
SetCollectionForDocument(csvReaderCurrentRecord[_collectionIndex], data);
}
continue;
}
if (_nestedPropertyDictionary != null && _nestedPropertyDictionary.TryGetValue(i, out var segments))
{
var nestedData = data;
for (var j = 0; ; j++)
{
//last segment holds the data
if (j == segments.Length - 1)
{
nestedData[segments[j]] = ParseValue(csvReaderCurrentRecord[i]);
break; //we are done
}
//Creating the objects along the path if needed e.g. Foo.Bar.Name will create the 'Bar' oject if needed
if (nestedData[segments[j]] == null)
{
var tmpRef = new DynamicJsonValue();
nestedData[segments[j]] = tmpRef;
nestedData = tmpRef;
}
//We need to advance into the nested object, since it is not the last segment it must be of type 'DynamicJsonValue'
else
{
nestedData = (DynamicJsonValue)nestedData[segments[j]];
}
}
continue;
}
data[csvReaderFieldHeaders[i]] = ParseValue(csvReaderCurrentRecord[i]);
}
if (_hasCollection == false)
{
SetCollectionForDocument(collection, data);
}
return(new DocumentItem
{
Document = new Document
{
Data = context.ReadObject(data, idStr),
Id = context.GetLazyString(idStr),
ChangeVector = string.Empty,
Flags = DocumentFlags.None,
NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler,
LastModified = _database.Time.GetUtcNow(),
},
Attachments = null
});
}
finally
{
foreach (var disposeMe in _disposibales)
{
disposeMe.Dispose();
}
_disposibales.Clear();
}
}
public unsafe void HandleConflictForDocument(
DocumentsOperationContext documentsContext,
string id,
string collection,
long lastModifiedTicks,
BlittableJsonReaderObject doc,
string changeVector,
DocumentFlags flags)
{
if (id.StartsWith(HiLoHandler.RavenHiloIdPrefix, StringComparison.OrdinalIgnoreCase))
{
HandleHiloConflict(documentsContext, id, doc, changeVector);
return;
}
if (TryResolveIdenticalDocument(
documentsContext,
id,
doc,
lastModifiedTicks,
changeVector))
{
return;
}
var lazyId = documentsContext.GetLazyString(id);
using (DocumentIdWorker.GetLower(documentsContext.Allocator, lazyId, out var loweredKey))
{
var conflictedDoc = new DocumentConflict
{
Doc = doc,
Collection = documentsContext.GetLazyStringForFieldWithCaching(
collection ??
CollectionName.GetCollectionName(doc)
),
LastModified = new DateTime(lastModifiedTicks),
LowerId = documentsContext.AllocateStringValue(null, loweredKey.Content.Ptr, loweredKey.Content.Length),
Id = lazyId,
ChangeVector = changeVector,
Flags = flags
};
if (TryResolveConflictByScript(
documentsContext,
conflictedDoc))
{
return;
}
if (_conflictResolver.ConflictSolver?.ResolveToLatest ?? true)
{
var conflicts = new List <DocumentConflict>
{
conflictedDoc.Clone()
};
conflicts.AddRange(_database.DocumentsStorage.ConflictsStorage.GetConflictsFor(documentsContext, id));
var localDocumentTuple = _database.DocumentsStorage.GetDocumentOrTombstone(documentsContext, id, false);
var local = DocumentConflict.From(documentsContext, localDocumentTuple.Document) ?? DocumentConflict.From(localDocumentTuple.Tombstone);
if (local != null)
{
conflicts.Add(local);
}
var resolved = _conflictResolver.ResolveToLatest(conflicts);
_conflictResolver.PutResolvedDocument(documentsContext, resolved, resolvedToLatest: true, conflictedDoc);
return;
}
_database.DocumentsStorage.ConflictsStorage.AddConflict(documentsContext, id, lastModifiedTicks, doc, changeVector, collection, flags);
}
}
public JsValue PutDocument(JsValue self, JsValue[] args)
{
string changeVector = null;
if (args.Length != 2 && args.Length != 3)
{
throw new InvalidOperationException("put(id, doc, changeVector) must be called with called with 2 or 3 arguments only");
}
AssertValidDatabaseContext();
AssertNotReadOnly();
if (args[0].IsString() == false && args[0].IsNull() == false && args[0].IsUndefined() == false)
{
AssertValidId();
}
var id = args[0].IsNull() || args[0].IsUndefined() ? null : args[0].AsString();
if (args[1].IsObject() == false)
{
throw new InvalidOperationException(
$"Created document must be a valid object which is not null or empty. Document ID: '{id}'.");
}
PutOrDeleteCalled = true;
if (args.Length == 3)
{
if (args[2].IsString())
{
changeVector = args[2].AsString();
}
else if (args[2].IsNull() == false && args[0].IsUndefined() == false)
{
throw new InvalidOperationException(
$"The change vector must be a string or null. Document ID: '{id}'.");
}
}
BlittableJsonReaderObject reader = null;
try
{
reader = JsBlittableBridge.Translate(_jsonCtx, ScriptEngine, args[1].AsObject(), usageMode: BlittableJsonDocumentBuilder.UsageMode.ToDisk);
var put = _database.DocumentsStorage.Put(
_docsCtx,
id,
_docsCtx.GetLazyString(changeVector),
reader,
nonPersistentFlags: NonPersistentDocumentFlags.ResolveAttachmentsConflict
);
if (DebugMode)
{
DebugActions.PutDocument.Add(new DynamicJsonValue
{
["Id"] = put.Id,
["Data"] = reader
});
}
return(put.Id);
}
finally
{
if (DebugMode == false)
{
reader?.Dispose();
}
}
}
public void AddConflict(
DocumentsOperationContext context,
string id,
long lastModifiedTicks,
BlittableJsonReaderObject incomingDoc,
string incomingChangeVector,
string incomingTombstoneCollection,
DocumentFlags flags)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Adding conflict to {id} (Incoming change vector {incomingChangeVector})");
}
var tx = context.Transaction.InnerTransaction;
var conflictsTable = tx.OpenTable(ConflictsSchema, ConflictsSlice);
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
CollectionName collectionName;
// ReSharper disable once ArgumentsStyleLiteral
var existing = _documentsStorage.GetDocumentOrTombstone(context, id, throwOnConflict: false);
LazyStringValue lazyCollectionName;
if (existing.Document != null)
{
var existingDoc = existing.Document;
lazyCollectionName = CollectionName.GetLazyCollectionNameFrom(context, existingDoc.Data);
using (Slice.From(context.Allocator, existingDoc.ChangeVector, out var cv))
using (conflictsTable.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(idPtr);
tvb.Add(existingDoc.Data.BasePointer, existingDoc.Data.Size);
tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag()));
tvb.Add(lazyCollectionName.Buffer, lazyCollectionName.Size);
tvb.Add(existingDoc.LastModified.Ticks);
tvb.Add((int)existingDoc.Flags);
if (conflictsTable.Set(tvb))
{
Interlocked.Increment(ref ConflictsCount);
}
}
// we delete the data directly, without generating a tombstone, because we have a
// conflict instead
_documentsStorage.EnsureLastEtagIsPersisted(context, existingDoc.Etag);
collectionName = _documentsStorage.ExtractCollectionName(context, existingDoc.Id, existingDoc.Data);
//make sure that the relevant collection tree exists
var table = tx.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));
table.Delete(existingDoc.StorageId);
}
else if (existing.Tombstone != null)
{
var existingTombstone = existing.Tombstone;
using (Slice.From(context.Allocator, existingTombstone.ChangeVector, out var cv))
using (conflictsTable.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(idPtr);
tvb.Add(null, 0);
tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag()));
tvb.Add(existingTombstone.Collection.Buffer, existingTombstone.Collection.Size);
tvb.Add(existingTombstone.LastModified.Ticks);
tvb.Add((int)existingTombstone.Flags);
if (conflictsTable.Set(tvb))
{
Interlocked.Increment(ref ConflictsCount);
}
}
// we delete the data directly, without generating a tombstone, because we have a
// conflict instead
_documentsStorage.EnsureLastEtagIsPersisted(context, existingTombstone.Etag);
collectionName = _documentsStorage.GetCollection(existingTombstone.Collection, throwIfDoesNotExist: true);
var table = tx.OpenTable(TombstonesSchema, collectionName.GetTableName(CollectionTableType.Tombstones));
table.Delete(existingTombstone.StorageId);
}
else // has existing conflicts
{
collectionName = _documentsStorage.ExtractCollectionName(context, id, incomingDoc);
using (GetConflictsIdPrefix(context, lowerId, out Slice prefixSlice))
{
var conflicts = GetConflictsFor(context, prefixSlice);
foreach (var conflict in conflicts)
{
var conflictStatus = ChangeVectorUtils.GetConflictStatus(incomingChangeVector, conflict.ChangeVector);
switch (conflictStatus)
{
case ConflictStatus.Update:
DeleteConflictsFor(context, conflict.ChangeVector); // delete this, it has been subsumed
break;
case ConflictStatus.Conflict:
break; // we'll add this conflict if no one else also includes it
case ConflictStatus.AlreadyMerged:
return; // we already have a conflict that includes this version
default:
throw new ArgumentOutOfRangeException("Invalid conflict status " + conflictStatus);
}
}
}
}
var etag = _documentsStorage.GenerateNextEtag();
ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentDatabase.DbId, etag, ref context.LastDatabaseChangeVector);
byte *doc = null;
var docSize = 0;
if (incomingDoc != null) // can be null if it is a tombstone
{
doc = incomingDoc.BasePointer;
docSize = incomingDoc.Size;
lazyCollectionName = CollectionName.GetLazyCollectionNameFrom(context, incomingDoc);
}
else
{
lazyCollectionName = context.GetLazyString(incomingTombstoneCollection);
}
using (lazyCollectionName)
using (Slice.From(context.Allocator, incomingChangeVector, out var cv))
using (conflictsTable.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(idPtr);
tvb.Add(doc, docSize);
tvb.Add(Bits.SwapBytes(etag));
tvb.Add(lazyCollectionName.Buffer, lazyCollectionName.Size);
tvb.Add(lastModifiedTicks);
tvb.Add((int)flags);
if (conflictsTable.Set(tvb))
{
Interlocked.Increment(ref ConflictsCount);
}
}
context.Transaction.AddAfterCommitNotification(new DocumentChange
{
ChangeVector = incomingChangeVector,
CollectionName = collectionName.Name,
Id = id,
Type = DocumentChangeTypes.Conflict,
IsSystemDocument = false
});
}
}