private bool TryResolveConflictByScript( DocumentsOperationContext documentsContext, DocumentConflict conflict) { var collection = conflict.Collection; var hasScript = _conflictResolver.ScriptConflictResolversCache.TryGetValue(collection, out ScriptResolver scriptResolver); if (!hasScript || scriptResolver == null) { if (_log.IsInfoEnabled) { _log.Info($"Script not found to resolve the {collection} collection"); } return(false); } var conflictedDocs = new List <DocumentConflict>(documentsContext.DocumentDatabase.DocumentsStorage.ConflictsStorage.GetConflictsFor(documentsContext, conflict.Id)); if (conflictedDocs.Count == 0) { var relevantLocalDoc = documentsContext.DocumentDatabase.DocumentsStorage .GetDocumentOrTombstone( documentsContext, conflict.Id); if (relevantLocalDoc.Document != null) { conflictedDocs.Add(DocumentConflict.From(documentsContext, relevantLocalDoc.Document)); } else if (relevantLocalDoc.Tombstone != null) { conflictedDocs.Add(DocumentConflict.From(relevantLocalDoc.Tombstone)); } } if (conflictedDocs.Count == 0) { InvalidConflictWhenThereIsNone(conflict.Id); } conflictedDocs.Add(conflict.Clone()); if (_conflictResolver.TryResolveConflictByScriptInternal( documentsContext, scriptResolver, conflictedDocs, documentsContext.GetLazyString(collection), out var resolved)) { _conflictResolver.PutResolvedDocument(documentsContext, resolved, conflict); return(true); } return(false); }
public JsValue PutDocument(JsValue self, JsValue[] args) { string changeVector = null; if (args.Length != 2 && args.Length != 3) { throw new InvalidOperationException("put(id, doc, changeVector) must be called with called with 2 or 3 arguments only"); } AssertValidDatabaseContext(); AssertNotReadOnly(); if (args[0].IsString() == false && args[0].IsNull() == false && args[0].IsUndefined() == false) { AssertValidId(); } var id = args[0].IsNull() || args[0].IsUndefined() ? null : args[0].AsString(); if (args[1].IsObject() == false) { throw new InvalidOperationException( $"Created document must be a valid object which is not null or empty. Document ID: '{id}'."); } PutOrDeleteCalled = true; if (args.Length == 3) { if (args[2].IsString()) { changeVector = args[2].AsString(); } else if (args[2].IsNull() == false && args[0].IsUndefined() == false) { throw new InvalidOperationException( $"The change vector must be a string or null. Document ID: '{id}'."); } } if (DebugMode) { DebugActions.PutDocument.Add(id); } using (var reader = JsBlittableBridge.Translate(_context, ScriptEngine, args[1].AsObject(), usageMode: BlittableJsonDocumentBuilder.UsageMode.ToDisk)) { var put = _database.DocumentsStorage.Put(_context, id, _context.GetLazyString(changeVector), reader); return(put.Id); } }
protected unsafe void SetLazyStringValueFromString(DocumentsOperationContext context, out LazyStringValue prop) { prop = null; var size = *(int *)Reader.ReadExactly(sizeof(int)); if (size < 0) { return; } // This is a special (broken) case. // On the source it is stored as Slice in LSV format which is wrong(?) unlike the normal LSV the escaping position is kept before the value itself. // and therefore the escaping doesn't include in the LSV size. // Additionally, we are over-allocating so writing this value doesn't cause a failure (we look for the escaping after the value) // this also work, because we don't pass those values between contexts, if we need to do so, we convert it to string first. // TODO: this is inefficient, can skip string allocation prop = context.GetLazyString(Encoding.UTF8.GetString(Reader.ReadExactly(size), size)); }
private bool TryResolveUsingDefaultResolver( DocumentsOperationContext context, string id, string collection, string incomingChangeVector, BlittableJsonReaderObject doc) { if (_conflictResolver.ConflictSolver?.DatabaseResolverId == null) { return(false); } var conflicts = new List <DocumentConflict>(_database.DocumentsStorage.ConflictsStorage.GetConflictsFor(context, id)); var localDocumentTuple = _database.DocumentsStorage.GetDocumentOrTombstone(context, id, false); var localDoc = DocumentConflict.From(context, localDocumentTuple.Document) ?? DocumentConflict.From(localDocumentTuple.Tombstone); if (localDoc != null) { conflicts.Add(localDoc); } conflicts.Add(new DocumentConflict { ChangeVector = incomingChangeVector, Collection = context.GetLazyStringForFieldWithCaching( collection ?? CollectionName.GetCollectionName(id, doc)), Doc = doc, LowerId = context.GetLazyString(id) }); if (_conflictResolver.TryResolveUsingDefaultResolverInternal( context, _conflictResolver.ConflictSolver.DatabaseResolverId, conflicts, out var resolved)) { _conflictResolver.PutResolvedDocument(context, resolved); return(true); } return(false); }
private CollectionName ResolveConflictAndAddTombstone(DocumentsOperationContext context, string changeVector, IReadOnlyList <DocumentConflict> conflicts, out long etag) { var indexOfLargestEtag = FindIndexOfLargestEtagAndMergeChangeVectors(conflicts, out string mergedChangeVector); var latestConflict = conflicts[indexOfLargestEtag]; var collectionName = new CollectionName(latestConflict.Collection); using (DocumentIdWorker.GetSliceFromId(context, latestConflict.Id, out Slice lowerId)) { //note that CreateTombstone is also deleting conflicts etag = _documentsStorage.CreateTombstone(context, lowerId, latestConflict.Etag, collectionName, context.GetLazyString(mergedChangeVector), latestConflict.LastModified.Ticks, changeVector, latestConflict.Flags).Etag; } return(collectionName); }
public void HandleConflictForDocument( DocumentsOperationContext documentsContext, string id, string collection, long lastModifiedTicks, BlittableJsonReaderObject doc, string changeVector, string conflictedChangeVector, DocumentFlags flags) { if (id.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase)) { HandleHiloConflict(documentsContext, id, doc, changeVector); return; } if (TryResolveIdenticalDocument( documentsContext, id, doc, lastModifiedTicks, changeVector)) { return; } var conflictedDoc = new DocumentConflict { Doc = doc, Collection = documentsContext.GetLazyStringForFieldWithCaching( collection ?? CollectionName.GetCollectionName(doc) ), LastModified = new DateTime(lastModifiedTicks), LowerId = documentsContext.GetLazyString(id), Id = documentsContext.GetLazyString(id), ChangeVector = changeVector }; if (TryResolveConflictByScript( documentsContext, conflictedDoc)) { return; } if (_conflictResolver.ConflictSolver?.ResolveToLatest == true) { if (conflictedChangeVector == null) //precaution { throw new InvalidOperationException( "Detected conflict on replication, but could not figure out conflicted vector. This is not supposed to happen and is likely a bug."); } var conflicts = new List <DocumentConflict> { conflictedDoc.Clone() }; conflicts.AddRange(documentsContext.DocumentDatabase.DocumentsStorage.ConflictsStorage.GetConflictsFor( documentsContext, id)); var localDocumentTuple = documentsContext.DocumentDatabase.DocumentsStorage.GetDocumentOrTombstone(documentsContext, id, false); var local = DocumentConflict.From(documentsContext, localDocumentTuple.Document) ?? DocumentConflict.From(localDocumentTuple.Tombstone); if (local != null) { conflicts.Add(local); } var resolved = _conflictResolver.ResolveToLatest(documentsContext, conflicts); _conflictResolver.PutResolvedDocument(documentsContext, resolved, conflictedDoc); return; } _database.DocumentsStorage.ConflictsStorage.AddConflict(documentsContext, id, lastModifiedTicks, doc, changeVector, collection, flags); }
private bool TryResolveConflictByScript( DocumentsOperationContext documentsContext, string id, string incomingChangeVector, BlittableJsonReaderObject doc) { var collection = CollectionName.GetCollectionName(id, doc); var hasScript = _conflictResolver.ScriptConflictResolversCache.TryGetValue(collection, out ScriptResolver scriptResolver); if (!hasScript || scriptResolver == null) { if (_log.IsInfoEnabled) { _log.Info($"Script not found to resolve the {collection} collection"); } return(false); } var conflictedDocs = new List <DocumentConflict>(documentsContext.DocumentDatabase.DocumentsStorage.ConflictsStorage.GetConflictsFor(documentsContext, id)); if (conflictedDocs.Count == 0) { var relevantLocalDoc = documentsContext.DocumentDatabase.DocumentsStorage .GetDocumentOrTombstone( documentsContext, id); if (relevantLocalDoc.Document != null) { conflictedDocs.Add(DocumentConflict.From(documentsContext, relevantLocalDoc.Document)); } else if (relevantLocalDoc.Tombstone != null) { conflictedDocs.Add(DocumentConflict.From(relevantLocalDoc.Tombstone)); } } if (conflictedDocs.Count == 0) { InvalidConflictWhenThereIsNone(id); } conflictedDocs.Add(new DocumentConflict { LowerId = conflictedDocs[0].LowerId, Id = conflictedDocs[0].Id, Collection = documentsContext.GetLazyStringForFieldWithCaching(collection), ChangeVector = incomingChangeVector, Doc = doc }); if (_conflictResolver.TryResolveConflictByScriptInternal( documentsContext, scriptResolver, conflictedDocs, documentsContext.GetLazyString(collection), out var resolved)) { _conflictResolver.PutResolvedDocument(documentsContext, resolved); return(true); } return(false); }
private DocumentItem ConvertRecordToDocumentItem(DocumentsOperationContext context, string[] csvReaderCurrentRecord, string[] csvReaderFieldHeaders, string collection) { try { var idStr = _hasId ? csvReaderCurrentRecord[_idIndex] : _hasCollection ? $"{csvReaderCurrentRecord[_collectionIndex]}/" : $"{collection}/"; var data = new DynamicJsonValue(); for (int i = 0; i < csvReaderFieldHeaders.Length; i++) { //ignoring reserved properties if (csvReaderFieldHeaders[i][0] == '@') { if (_hasCollection && i == _collectionIndex) { SetCollectionForDocument(csvReaderCurrentRecord[_collectionIndex], data); } continue; } if (_nestedPropertyDictionary != null && _nestedPropertyDictionary.TryGetValue(i, out var segments)) { var nestedData = data; for (var j = 0; ; j++) { //last segment holds the data if (j == segments.Length - 1) { nestedData[segments[j]] = ParseValue(csvReaderCurrentRecord[i]); break; //we are done } //Creating the objects along the path if needed e.g. Foo.Bar.Name will create the 'Bar' oject if needed if (nestedData[segments[j]] == null) { var tmpRef = new DynamicJsonValue(); nestedData[segments[j]] = tmpRef; nestedData = tmpRef; } //We need to advance into the nested object, since it is not the last segment it must be of type 'DynamicJsonValue' else { nestedData = (DynamicJsonValue)nestedData[segments[j]]; } } continue; } data[csvReaderFieldHeaders[i]] = ParseValue(csvReaderCurrentRecord[i]); } if (_hasCollection == false) { SetCollectionForDocument(collection, data); } return(new DocumentItem { Document = new Document { Data = context.ReadObject(data, idStr), Id = context.GetLazyString(idStr), ChangeVector = string.Empty, Flags = DocumentFlags.None, NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler, LastModified = _database.Time.GetUtcNow(), }, Attachments = null }); } finally { foreach (var disposeMe in _disposibales) { disposeMe.Dispose(); } _disposibales.Clear(); } }
public unsafe void HandleConflictForDocument( DocumentsOperationContext documentsContext, string id, string collection, long lastModifiedTicks, BlittableJsonReaderObject doc, string changeVector, DocumentFlags flags) { if (id.StartsWith(HiLoHandler.RavenHiloIdPrefix, StringComparison.OrdinalIgnoreCase)) { HandleHiloConflict(documentsContext, id, doc, changeVector); return; } if (TryResolveIdenticalDocument( documentsContext, id, doc, lastModifiedTicks, changeVector)) { return; } var lazyId = documentsContext.GetLazyString(id); using (DocumentIdWorker.GetLower(documentsContext.Allocator, lazyId, out var loweredKey)) { var conflictedDoc = new DocumentConflict { Doc = doc, Collection = documentsContext.GetLazyStringForFieldWithCaching( collection ?? CollectionName.GetCollectionName(doc) ), LastModified = new DateTime(lastModifiedTicks), LowerId = documentsContext.AllocateStringValue(null, loweredKey.Content.Ptr, loweredKey.Content.Length), Id = lazyId, ChangeVector = changeVector, Flags = flags }; if (TryResolveConflictByScript( documentsContext, conflictedDoc)) { return; } if (_conflictResolver.ConflictSolver?.ResolveToLatest ?? true) { var conflicts = new List <DocumentConflict> { conflictedDoc.Clone() }; conflicts.AddRange(_database.DocumentsStorage.ConflictsStorage.GetConflictsFor(documentsContext, id)); var localDocumentTuple = _database.DocumentsStorage.GetDocumentOrTombstone(documentsContext, id, false); var local = DocumentConflict.From(documentsContext, localDocumentTuple.Document) ?? DocumentConflict.From(localDocumentTuple.Tombstone); if (local != null) { conflicts.Add(local); } var resolved = _conflictResolver.ResolveToLatest(conflicts); _conflictResolver.PutResolvedDocument(documentsContext, resolved, resolvedToLatest: true, conflictedDoc); return; } _database.DocumentsStorage.ConflictsStorage.AddConflict(documentsContext, id, lastModifiedTicks, doc, changeVector, collection, flags); } }
public JsValue PutDocument(JsValue self, JsValue[] args) { string changeVector = null; if (args.Length != 2 && args.Length != 3) { throw new InvalidOperationException("put(id, doc, changeVector) must be called with called with 2 or 3 arguments only"); } AssertValidDatabaseContext(); AssertNotReadOnly(); if (args[0].IsString() == false && args[0].IsNull() == false && args[0].IsUndefined() == false) { AssertValidId(); } var id = args[0].IsNull() || args[0].IsUndefined() ? null : args[0].AsString(); if (args[1].IsObject() == false) { throw new InvalidOperationException( $"Created document must be a valid object which is not null or empty. Document ID: '{id}'."); } PutOrDeleteCalled = true; if (args.Length == 3) { if (args[2].IsString()) { changeVector = args[2].AsString(); } else if (args[2].IsNull() == false && args[0].IsUndefined() == false) { throw new InvalidOperationException( $"The change vector must be a string or null. Document ID: '{id}'."); } } BlittableJsonReaderObject reader = null; try { reader = JsBlittableBridge.Translate(_jsonCtx, ScriptEngine, args[1].AsObject(), usageMode: BlittableJsonDocumentBuilder.UsageMode.ToDisk); var put = _database.DocumentsStorage.Put( _docsCtx, id, _docsCtx.GetLazyString(changeVector), reader, nonPersistentFlags: NonPersistentDocumentFlags.ResolveAttachmentsConflict ); if (DebugMode) { DebugActions.PutDocument.Add(new DynamicJsonValue { ["Id"] = put.Id, ["Data"] = reader }); } return(put.Id); } finally { if (DebugMode == false) { reader?.Dispose(); } } }
public void AddConflict( DocumentsOperationContext context, string id, long lastModifiedTicks, BlittableJsonReaderObject incomingDoc, string incomingChangeVector, string incomingTombstoneCollection, DocumentFlags flags) { if (_logger.IsInfoEnabled) { _logger.Info($"Adding conflict to {id} (Incoming change vector {incomingChangeVector})"); } var tx = context.Transaction.InnerTransaction; var conflictsTable = tx.OpenTable(ConflictsSchema, ConflictsSlice); using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { CollectionName collectionName; // ReSharper disable once ArgumentsStyleLiteral var existing = _documentsStorage.GetDocumentOrTombstone(context, id, throwOnConflict: false); LazyStringValue lazyCollectionName; if (existing.Document != null) { var existingDoc = existing.Document; lazyCollectionName = CollectionName.GetLazyCollectionNameFrom(context, existingDoc.Data); using (Slice.From(context.Allocator, existingDoc.ChangeVector, out var cv)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(idPtr); tvb.Add(existingDoc.Data.BasePointer, existingDoc.Data.Size); tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag())); tvb.Add(lazyCollectionName.Buffer, lazyCollectionName.Size); tvb.Add(existingDoc.LastModified.Ticks); tvb.Add((int)existingDoc.Flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } // we delete the data directly, without generating a tombstone, because we have a // conflict instead _documentsStorage.EnsureLastEtagIsPersisted(context, existingDoc.Etag); collectionName = _documentsStorage.ExtractCollectionName(context, existingDoc.Id, existingDoc.Data); //make sure that the relevant collection tree exists var table = tx.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents)); table.Delete(existingDoc.StorageId); } else if (existing.Tombstone != null) { var existingTombstone = existing.Tombstone; using (Slice.From(context.Allocator, existingTombstone.ChangeVector, out var cv)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(idPtr); tvb.Add(null, 0); tvb.Add(Bits.SwapBytes(_documentsStorage.GenerateNextEtag())); tvb.Add(existingTombstone.Collection.Buffer, existingTombstone.Collection.Size); tvb.Add(existingTombstone.LastModified.Ticks); tvb.Add((int)existingTombstone.Flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } // we delete the data directly, without generating a tombstone, because we have a // conflict instead _documentsStorage.EnsureLastEtagIsPersisted(context, existingTombstone.Etag); collectionName = _documentsStorage.GetCollection(existingTombstone.Collection, throwIfDoesNotExist: true); var table = tx.OpenTable(TombstonesSchema, collectionName.GetTableName(CollectionTableType.Tombstones)); table.Delete(existingTombstone.StorageId); } else // has existing conflicts { collectionName = _documentsStorage.ExtractCollectionName(context, id, incomingDoc); using (GetConflictsIdPrefix(context, lowerId, out Slice prefixSlice)) { var conflicts = GetConflictsFor(context, prefixSlice); foreach (var conflict in conflicts) { var conflictStatus = ChangeVectorUtils.GetConflictStatus(incomingChangeVector, conflict.ChangeVector); switch (conflictStatus) { case ConflictStatus.Update: DeleteConflictsFor(context, conflict.ChangeVector); // delete this, it has been subsumed break; case ConflictStatus.Conflict: break; // we'll add this conflict if no one else also includes it case ConflictStatus.AlreadyMerged: return; // we already have a conflict that includes this version default: throw new ArgumentOutOfRangeException("Invalid conflict status " + conflictStatus); } } } } var etag = _documentsStorage.GenerateNextEtag(); ChangeVectorUtils.TryUpdateChangeVector(_documentDatabase.ServerStore.NodeTag, _documentDatabase.DbId, etag, ref context.LastDatabaseChangeVector); byte *doc = null; var docSize = 0; if (incomingDoc != null) // can be null if it is a tombstone { doc = incomingDoc.BasePointer; docSize = incomingDoc.Size; lazyCollectionName = CollectionName.GetLazyCollectionNameFrom(context, incomingDoc); } else { lazyCollectionName = context.GetLazyString(incomingTombstoneCollection); } using (lazyCollectionName) using (Slice.From(context.Allocator, incomingChangeVector, out var cv)) using (conflictsTable.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(idPtr); tvb.Add(doc, docSize); tvb.Add(Bits.SwapBytes(etag)); tvb.Add(lazyCollectionName.Buffer, lazyCollectionName.Size); tvb.Add(lastModifiedTicks); tvb.Add((int)flags); if (conflictsTable.Set(tvb)) { Interlocked.Increment(ref ConflictsCount); } } context.Transaction.AddAfterCommitNotification(new DocumentChange { ChangeVector = incomingChangeVector, CollectionName = collectionName.Name, Id = id, Type = DocumentChangeTypes.Conflict, IsSystemDocument = false }); } }