internal static BlittableJsonReaderObject EnsureLowerCasedIndexIdProperty(DocumentsOperationContext context, BlittableJsonReaderObject json, ElasticSearchIndexWithRecords index) { if (json.TryGet(index.DocumentIdProperty, out LazyStringValue idProperty)) { using (var old = json) { json.Modifications = new DynamicJsonValue(json) { [index.DocumentIdProperty] = LowerCaseDocumentIdProperty(idProperty) }; json = context.ReadObject(json, "es-etl-load"); } } else if (json.Modifications != null) { // document id property was not added by user, so we inserted the lowercased id in ElasticSearchDocumentTransformer.LoadToFunction #if DEBUG var docIdProperty = json.Modifications.Properties.First(x => x.Name == index.DocumentIdProperty); Debug.Assert(docIdProperty.Value.ToString() == docIdProperty.Value.ToString().ToLowerInvariant()); #endif json = context.ReadObject(json, "es-etl-load"); } return(json); }
protected override long ExecuteCmd(DocumentsOperationContext context) { var hiLoDocumentId = RavenHiloIdPrefix + Key; var prefix = Key + Separator; var newDoc = new DynamicJsonValue(); BlittableJsonReaderObject hiloDocReader = null; try { try { hiloDocReader = Database.DocumentsStorage.Get(context, hiLoDocumentId)?.Data; } catch (DocumentConflictException e) { throw new InvalidDataException("Failed to fetch HiLo document due to a conflict on the document. " + "This shouldn't happen, since it this conflict should've been resolved during replication. " + "This exception should not happen and is likely a bug.", e); } if (hiloDocReader == null) { OldMax = LastRangeMax; newDoc["Max"] = OldMax + Capacity; newDoc[Constants.Documents.Metadata.Key] = new DynamicJsonValue { [Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection }; using (var freshHilo = context.ReadObject(newDoc, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk)) Database.DocumentsStorage.Put(context, hiLoDocumentId, null, freshHilo); } else { hiloDocReader.TryGet("Max", out long oldMax); OldMax = Math.Max(oldMax, LastRangeMax); hiloDocReader.Modifications = new DynamicJsonValue(hiloDocReader) { ["Max"] = OldMax + Capacity }; using (var freshHilo = context.ReadObject(hiloDocReader, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk)) Database.DocumentsStorage.Put(context, hiLoDocumentId, null, freshHilo); } Prefix = prefix; } finally { hiloDocReader?.Dispose(); } return(1); }
protected override long ExecuteCmd(DocumentsOperationContext context) { var hiLoDocumentId = RavenHiloIdPrefix + Key; var document = Database.DocumentsStorage.Get(context, hiLoDocumentId); if (document == null) { return(1); } document.Data.TryGet("Max", out long oldMax); if (oldMax != End || Last > oldMax) { return(1); } document.Data.Modifications = new DynamicJsonValue { ["Max"] = Last }; using (var hiloReader = context.ReadObject(document.Data, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk)) { Database.DocumentsStorage.Put(context, hiLoDocumentId, null, hiloReader); } return(1); }
public static LegacyAttachmentDetails GenerateLegacyAttachmentDetails( DocumentsOperationContext context, Stream decodedStream, string key, BlittableJsonReaderObject metadata, ref DocumentItem.AttachmentStream attachment) { var stream = attachment.Stream; var hash = AsyncHelpers.RunSync(() => AttachmentsStorageHelper.CopyStreamToFileAndCalculateHash(context, decodedStream, stream, CancellationToken.None)); attachment.Stream.Flush(); var lazyHash = context.GetLazyString(hash); attachment.Base64HashDispose = Slice.External(context.Allocator, lazyHash, out attachment.Base64Hash); var tag = $"{DummyDocumentPrefix}{key}{RecordSeparator}d{RecordSeparator}{key}{RecordSeparator}{hash}{RecordSeparator}"; var lazyTag = context.GetLazyString(tag); attachment.TagDispose = Slice.External(context.Allocator, lazyTag, out attachment.Tag); var id = GetLegacyAttachmentId(key); var lazyId = context.GetLazyString(id); attachment.Data = context.ReadObject(metadata, id); return(new LegacyAttachmentDetails { Id = lazyId, Hash = hash, Key = key, Size = attachment.Stream.Length, Tag = tag, Metadata = attachment.Data }); }
public static BlittableJsonReaderObject WriteDummyDocumentForAttachment(DocumentsOperationContext context, LegacyAttachmentDetails details) { var attachment = new DynamicJsonValue { ["Name"] = details.Key, ["Hash"] = details.Hash, ["ContentType"] = string.Empty, ["Size"] = details.Size, }; var attachments = new DynamicJsonArray(); attachments.Add(attachment); var metadata = new DynamicJsonValue { [Constants.Documents.Metadata.Collection] = "@files", [Constants.Documents.Metadata.Attachments] = attachments, [Constants.Documents.Metadata.LegacyAttachmentsMetadata] = details.Metadata }; var djv = new DynamicJsonValue { [Constants.Documents.Metadata.Key] = metadata, }; return(context.ReadObject(djv, details.Id)); }
private static void HandleResultsWithoutSelect <TResult>( DocumentsOperationContext documentsContext, List <Match> matchResults, TResult final) where TResult : QueryResultServerSide <Document> { foreach (var match in matchResults) { if (match.Empty) { continue; } if (match.Count == 1) //if we don't have multiple results in each row, we can "flatten" the row { final.AddResult(match.GetFirstResult()); continue; } var resultAsJson = new DynamicJsonValue(); match.PopulateVertices(resultAsJson); var result = new Document { Data = documentsContext.ReadObject(resultAsJson, "graph/result"), }; final.AddResult(result); } }
private void WriteAllEnvs(BlittableJsonTextWriter writer, DocumentsOperationContext context) { var envs = Database.GetAllStoragesEnvironment(); bool first = true; foreach (var env in envs) { if (env == null) { continue; } if (!first) { writer.WriteComma(); } first = false; writer.WriteStartObject(); writer.WritePropertyName("Environment"); writer.WriteString(env.Name); writer.WriteComma(); writer.WritePropertyName("Type"); writer.WriteString(env.Type.ToString()); writer.WriteComma(); var djv = (DynamicJsonValue)TypeConverter.ToBlittableSupportedType(GetDetailedReport(env, false)); writer.WritePropertyName("Report"); writer.WriteObject(context.ReadObject(djv, env.Name)); writer.WriteEndObject(); } }
public override void Execute(DocumentsOperationContext context, RavenTransaction tx) { var hiLoDocumentKey = RavenKeyGeneratorsHilo + Key; var document = Database.DocumentsStorage.Get(context, hiLoDocumentKey); if (document == null) { return; } long oldMax; document.Data.TryGet("Max", out oldMax); if (oldMax != End || Last > oldMax) { return; } document.Data.Modifications = new DynamicJsonValue() { ["Max"] = Last, }; using (var hiloReader = context.ReadObject(document.Data, hiLoDocumentKey, BlittableJsonDocumentBuilder.UsageMode.ToDisk)) { Database.DocumentsStorage.Put(context, hiLoDocumentKey, null, hiloReader); } }
public void AddIncludesToResult(TimeSeriesRangeResult rangeResult) { if (rangeResult == null || _includes?.Properties.Count > 0 == false) { return; } rangeResult.Includes = _context.ReadObject(_includes, "TimeSeriesRangeIncludes/" + _docId); }
public override int Execute(DocumentsOperationContext context) { var hiLoDocumentId = RavenIdGeneratorsHilo + Key; var prefix = Key + Separator; long oldMax = 0; var newDoc = new DynamicJsonValue(); BlittableJsonReaderObject hiloDocReader = null; try { try { hiloDocReader = Database.DocumentsStorage.Get(context, hiLoDocumentId)?.Data; } catch (DocumentConflictException e) { throw new InvalidDataException("Failed to fetch HiLo document due to a conflict on the document. " + "This shouldn't happen, since it this conflict should've been resolved during replication. " + "This exception should not happen and is likely a bug.", e); } if (hiloDocReader != null) { var prop = new BlittableJsonReaderObject.PropertyDetails(); foreach (var propertyId in hiloDocReader.GetPropertiesByInsertionOrder()) { hiloDocReader.GetPropertyByIndex(propertyId, ref prop); if (prop.Name == "Max") { oldMax = (long)prop.Value; continue; } newDoc[prop.Name] = prop.Value; } } oldMax = Math.Max(oldMax, LastRangeMax); newDoc["Max"] = oldMax + Capacity; using (var freshHilo = context.ReadObject(newDoc, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk)) Database.DocumentsStorage.Put(context, hiLoDocumentId, null, freshHilo); OldMax = oldMax; Prefix = prefix; } finally { hiloDocReader?.Dispose(); } return(1); }
private void WriteLastEtag(DocumentsOperationContext context) { var key = Constants.SqlReplication.RavenSqlReplicationStatusPrefix + ReplicationUniqueName; var document = context.ReadObject(new DynamicJsonValue { ["Name"] = ReplicationUniqueName, ["LastReplicatedEtag"] = Statistics.LastReplicatedEtag, ["LastTombstonesEtag"] = Statistics.LastTombstonesEtag, }, key, BlittableJsonDocumentBuilder.UsageMode.ToDisk); _database.DocumentsStorage.Put(context, key, null, document); }
public void Delete(DocumentsOperationContext context, string id, Slice lowerId, CollectionName collectionName, string changeVector, long lastModifiedTicks, NonPersistentDocumentFlags nonPersistentFlags, DocumentFlags flags) { using (DocumentIdWorker.GetStringPreserveCase(context, id, out Slice idPtr)) { var deleteRevisionDocument = context.ReadObject(new DynamicJsonValue { [Constants.Documents.Metadata.Key] = new DynamicJsonValue { [Constants.Documents.Metadata.Collection] = collectionName.Name } }, "RevisionsBin"); Delete(context, lowerId, idPtr, id, collectionName, deleteRevisionDocument, changeVector, lastModifiedTicks, nonPersistentFlags, flags); } }
private BlittableJsonReaderObject GetCleanMetadata(BlittableJsonReaderObject metadata, DocumentsOperationContext context) { metadata.Modifications = new DynamicJsonValue(metadata); metadata.Modifications.Remove("Origin"); metadata.Modifications.Remove("Raven-Synchronization-Version"); metadata.Modifications.Remove("Raven-Synchronization-Source"); metadata.Modifications.Remove("Creation-Date"); metadata.Modifications.Remove("Raven-Creation-Date"); metadata.Modifications.Remove("Raven-Synchronization-History"); metadata.Modifications.Remove("RavenFS-Size"); metadata.Modifications.Remove("Last-Modified"); metadata.Modifications.Remove("Raven-Last-Modified"); metadata.Modifications.Remove("Content-MD5"); metadata.Modifications.Remove("ETag"); return(context.ReadObject(metadata, MigrationStateKey)); }
public virtual string PutDocument(string key, JsValue document, JsValue metadata, JsValue etagJs, Engine engine) { if (document.IsObject() == false) { throw new InvalidOperationException( $"Created document must be a valid object which is not null or empty. Document key: '{key}'."); } long?etag = null; if (etagJs.IsNumber()) { etag = (long)etagJs.AsNumber(); } else if (etagJs.IsNull() == false && etagJs.IsUndefined() == false && etagJs.ToString() != "None") { throw new InvalidOperationException($"Invalid ETag value for document '{key}'"); } var data = ToBlittable(document.AsObject()); if (metadata.IsObject()) { data["@metadata"] = ToBlittable(metadata.AsObject()); } if (DebugMode) { DebugActions.PutDocument.Add(new DynamicJsonValue { ["Key"] = key, ["Etag"] = etag, ["Data"] = data, }); } var dataReader = _context.ReadObject(data, key, BlittableJsonDocumentBuilder.UsageMode.ToDisk); var put = _database.DocumentsStorage.Put(_context, key, etag, dataReader); return(put.Key); }
private void WriteIntermidiateResults(List <GraphQueryRunner.Match> matches) { _writer.WritePropertyName("Results"); _writer.WriteStartArray(); var first = true; foreach (var match in matches) { if (first == false) { _writer.WriteComma(); } first = false; var djv = new DynamicJsonValue(); match.PopulateVertices(djv); _writer.WriteObject(_ctx.ReadObject(djv, null)); } _writer.WriteEndArray(); }
public virtual PatchResultData Apply(DocumentsOperationContext context, Document document, PatchRequest patch) { if (document == null) { return(null); } if (string.IsNullOrEmpty(patch.Script)) { throw new InvalidOperationException("Patch script must be non-null and not empty"); } var scope = ApplySingleScript(context, document, false, patch); var modifiedDocument = context.ReadObject(scope.ToBlittable(scope.PatchObject.AsObject()), document.Key); /* TODO: Should not use BlittableJsonDocumentBuilder.UsageMode.ToDisk? */ return(new PatchResultData { ModifiedDocument = modifiedDocument ?? document.Data, DebugInfo = scope.DebugInfo, }); }
private JsValue GetMetadata(JsValue self, JsValue[] args) { if (args.Length != 1 || !(args[0].AsObject() is BlittableObjectInstance boi)) { throw new InvalidOperationException("getMetadata(doc) must be called with a single entity argument"); } if (!(boi.Blittable[Constants.Documents.Metadata.Key] is BlittableJsonReaderObject metadata)) { return(JsValue.Null); } metadata.Modifications = new DynamicJsonValue { [Constants.Documents.Metadata.ChangeVector] = boi.ChangeVector, [Constants.Documents.Metadata.Id] = boi.DocumentId, [Constants.Documents.Metadata.LastModified] = boi.LastModified, }; metadata = _context.ReadObject(metadata, boi.DocumentId); return(TranslateToJs(ScriptEngine, _context, metadata)); }
public void Put(DocumentsOperationContext context, string id, BlittableJsonReaderObject document, DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags, string changeVector, long lastModifiedTicks, RevisionsCollectionConfiguration configuration = null, CollectionName collectionName = null) { Debug.Assert(changeVector != null, "Change vector must be set"); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); if (collectionName == null) { collectionName = _database.DocumentsStorage.ExtractCollectionName(context, id, document); } using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { var fromSmuggler = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler; var fromReplication = (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) == NonPersistentDocumentFlags.FromReplication; var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName); // We want the revision's attachments to have a lower etag than the revision itself if ((flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments && fromSmuggler == false) { using (Slice.From(context.Allocator, changeVector, out Slice changeVectorSlice)) { if (table.VerifyKeyExists(changeVectorSlice) == false) { _documentsStorage.AttachmentsStorage.RevisionAttachments(context, lowerId, changeVectorSlice); } } } if (fromReplication) { void PutFromRevisionIfChangeVectorIsGreater() { bool hasDoc; TableValueReader tvr; try { hasDoc = _documentsStorage.GetTableValueReaderForDocument(context, lowerId, throwOnConflict: true, tvr: out tvr); } catch (DocumentConflictException) { // Do not modify the document. return; } if (hasDoc == false) { PutFromRevision(); return; } var docChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref tvr); if (ChangeVectorUtils.GetConflictStatus(changeVector, docChangeVector) == ConflictStatus.Update) { PutFromRevision(); } void PutFromRevision() { _documentsStorage.Put(context, id, null, document, lastModifiedTicks, changeVector, flags & ~DocumentFlags.Revision, nonPersistentFlags | NonPersistentDocumentFlags.FromRevision); } } PutFromRevisionIfChangeVectorIsGreater(); } flags |= DocumentFlags.Revision; var data = context.ReadObject(document, id); var newEtag = _database.DocumentsStorage.GenerateNextEtag(); var newEtagSwapBytes = Bits.SwapBytes(newEtag); using (table.Allocate(out TableValueBuilder tvb)) using (Slice.From(context.Allocator, changeVector, out var cv)) { tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(newEtagSwapBytes); tvb.Add(idPtr); tvb.Add(data.BasePointer, data.Size); tvb.Add((int)flags); tvb.Add(newEtagSwapBytes); tvb.Add(lastModifiedTicks); tvb.Add(context.GetTransactionMarker()); var isNew = table.Set(tvb); if (isNew == false) { // It might be just an update from replication as we call this twice, both for the doc delete and for deleteRevision. return; } } if (configuration == null) { configuration = GetRevisionsConfiguration(collectionName.Name); } DeleteOldRevisions(context, table, lowerId, collectionName, configuration, nonPersistentFlags, changeVector); } }
private void DeleteCounter(UpdateStep step, LazyStringValue tombstoneKey, DocumentsOperationContext context) { var(docId, counterName) = ExtractDocIdAndNameFromCounterTombstone(context, tombstoneKey); using (docId) using (counterName) using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, docId, out Slice lowerId, out _)) { string collection = null; var docsTable = new Table(DocsSchema, step.ReadTx); if (docsTable.ReadByKey(lowerId, out var tvr)) { using (var doc = new BlittableJsonReaderObject(tvr.Read((int)DocumentsTable.Data, out int size), size, context)) { collection = CollectionName.GetCollectionName(doc); } } var collectionName = new CollectionName(collection); var table = step.DocumentsStorage.CountersStorage.GetCountersTable(step.WriteTx, collectionName); if (table.ReadByKey(lowerId, out var existing) == false) { return; } // (int)CountersTable.Data = 3 var data = new BlittableJsonReaderObject(existing.Read(3, out int oldSize), oldSize, context); if (data.TryGet(CountersStorage.Values, out BlittableJsonReaderObject counters) == false || counters.TryGetMember(counterName, out object counterToDelete) == false || counterToDelete is LazyStringValue) // already deleted { return; } var deleteCv = step.DocumentsStorage.CountersStorage.GenerateDeleteChangeVectorFromRawBlob(data, counterToDelete as BlittableJsonReaderObject.RawBlob); counters.Modifications = new DynamicJsonValue(counters) { [counterName] = deleteCv }; using (var old = data) { data = context.ReadObject(data, null, BlittableJsonDocumentBuilder.UsageMode.ToDisk); } var newEtag = step.DocumentsStorage.GenerateNextEtag(); var newChangeVector = ChangeVectorUtils.NewChangeVector(step.DocumentsStorage.DocumentDatabase.ServerStore.NodeTag, newEtag, _dbId); using (data) using (Slice.From(context.Allocator, newChangeVector, out var cv)) using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(Bits.SwapBytes(newEtag)); tvb.Add(cv); tvb.Add(data.BasePointer, data.Size); tvb.Add(collectionSlice); tvb.Add(context.GetTransactionMarker()); table.Set(tvb); } } }
public PutOperationResults PutDocument(DocumentsOperationContext context, string id, string expectedChangeVector, BlittableJsonReaderObject document, long?lastModifiedTicks = null, string changeVector = null, DocumentFlags flags = DocumentFlags.None, NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None) { if (context.Transaction == null) { ThrowRequiresTransaction(); return(default(PutOperationResults)); // never hit } #if DEBUG var documentDebugHash = document.DebugHash; document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); #endif var newEtag = _documentsStorage.GenerateNextEtag(); var modifiedTicks = lastModifiedTicks ?? _documentDatabase.Time.GetUtcNow().Ticks; id = BuildDocumentId(id, newEtag, out bool knownNewId); using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { var collectionName = _documentsStorage.ExtractCollectionName(context, document); var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents)); var oldValue = default(TableValueReader); if (knownNewId == false) { // delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size); table.ReadByKey(lowerId, out oldValue); } BlittableJsonReaderObject oldDoc = null; if (oldValue.Pointer == null) { // expectedChangeVector being null means we don't care, and empty // means that it must be new if (string.IsNullOrEmpty(expectedChangeVector) == false) { ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector); } } else { // expectedChangeVector has special meaning here // null - means, don't care, don't check // "" / empty - means, must be new // anything else - must match exactly if (expectedChangeVector != null) { var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue); if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0) { ThrowConcurrentException(id, expectedChangeVector, oldChangeVector); } } oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context); var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc); if (oldCollectionName != collectionName) { ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName); } var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue); if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate && (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication) { if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments) { flags |= DocumentFlags.HasAttachments; } } } var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue); changeVector = result.ChangeVector; nonPersistentFlags |= result.NonPersistentFlags; if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved)) { flags |= DocumentFlags.Resolved; } if (collectionName.IsHiLo == false && (flags & DocumentFlags.Artificial) != DocumentFlags.Artificial) { if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags)) { #if DEBUG if (document.DebugHash != documentDebugHash) { throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " + "this is likely because you are trying to save a document that is already stored and was moved"); } #endif document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); #if DEBUG documentDebugHash = document.DebugHash; document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); AttachmentsStorage.AssertAttachments(document, flags); #endif } if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false && (flags.Contain(DocumentFlags.Resolved) || _documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null )) { var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document, ref flags, out RevisionsCollectionConfiguration configuration); if (shouldVersion) { _documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags, changeVector, modifiedTicks, configuration, collectionName); } } } using (Slice.From(context.Allocator, changeVector, out var cv)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(Bits.SwapBytes(newEtag)); tvb.Add(idPtr); tvb.Add(document.BasePointer, document.Size); tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(modifiedTicks); tvb.Add((int)flags); tvb.Add(context.GetTransactionMarker()); if (oldValue.Pointer == null) { table.Insert(tvb); } else { table.Update(oldValue.Id, tvb); } } if (collectionName.IsHiLo == false) { _documentsStorage.ExpirationStorage.Put(context, lowerId, document); } context.LastDatabaseChangeVector = changeVector; _documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1); _documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size); context.Transaction.AddAfterCommitNotification(new DocumentChange { ChangeVector = changeVector, CollectionName = collectionName.Name, Id = id, Type = DocumentChangeTypes.Put, }); #if DEBUG if (document.DebugHash != documentDebugHash) { throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " + "this is likely because you are trying to save a document that is already stored and was moved"); } document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); AttachmentsStorage.AssertAttachments(document, flags); #endif return(new PutOperationResults { Etag = newEtag, Id = id, Collection = collectionName, ChangeVector = changeVector, Flags = flags, LastModified = new DateTime(modifiedTicks) }); } }
internal Document ProjectFromMatch(Match match, JsonOperationContext context, CancellationToken token) { var result = new DynamicJsonValue(); result[Constants.Documents.Metadata.Key] = new DynamicJsonValue { [Constants.Documents.Metadata.Projection] = true }; var item = new Document(); foreach (var fieldToFetch in FieldsToFetch.Fields.Values) { object fieldVal; string key; if (fieldToFetch.QueryField?.ExpressionField?.Compound?.Contains("[]") ?? false) { key = fieldToFetch.QueryField.ExpressionField.Compound.LastOrDefault().Value; } else { key = fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value; } if (fieldToFetch.QueryField.Function != null) { var args = new object[fieldToFetch.FunctionArgs.Length + 1]; for (int i = 0; i < fieldToFetch.FunctionArgs.Length; i++) { var val = match.GetResult(fieldToFetch.FunctionArgs[i].ProjectedName); switch (val) { case Document doc: doc.EnsureMetadata(); args[i] = doc; break; case BlittableJsonReaderObject bjro: args[i] = bjro; break; case List <Match> matchList: CreateArgsFromMatchList(matchList, args, i); break; case MatchCollection matchCollection: CreateArgsFromMatchList(matchCollection, args, i); break; case string s: args[i] = s; break; default: args[i] = null; break; } } key = fieldToFetch.ProjectedName ?? (fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value); fieldVal = GetFunctionValue(fieldToFetch, null, args, token); var immediateResult = AddProjectionToResult(item, OneScore, FieldsToFetch, result, key, fieldVal); if (immediateResult.Document != null) { return(immediateResult.Document); } } else { var val = match.GetResult(fieldToFetch.QueryField.ExpressionField.Compound[0].Value); switch (val) { case Document d: { if (TryGetValue(fieldToFetch, d, null, null, null, null, out key, out fieldVal, token) == false) { continue; } d.EnsureMetadata(); var immediateResult = AddProjectionToResult(d, OneScore, FieldsToFetch, result, key, fieldVal); if (immediateResult.Document != null) { return(immediateResult.Document); } break; } case BlittableJsonReaderObject bjro: { var doc = new Document { Data = bjro }; if (TryGetValue(fieldToFetch, doc, null, null, null, null, out key, out fieldVal, token) == false) { continue; } doc.EnsureMetadata(); var immediateResult = AddProjectionToResult(doc, OneScore, FieldsToFetch, result, key, fieldVal); if (immediateResult.Document != null) { return(immediateResult.Document); } break; } case MatchCollection matches: var array = new DynamicJsonArray(); foreach (var m in matches) { var djv = new DynamicJsonValue(); m.PopulateVertices(djv); if (djv.Properties.Count == 0) { continue; } var matchJson = _context.ReadObject(djv, "graph/arg"); var doc = new Document { Data = matchJson }; if (TryGetValue(fieldToFetch, doc, null, null, null, null, out key, out fieldVal, token) == false) { continue; } doc.EnsureMetadata(); if (ReferenceEquals(doc, fieldVal)) { fieldVal = doc.Data; } array.Add(fieldVal); } result[key] = array; break; case string s: result[fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value] = s; break; default: result[fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value] = null; continue; } } } return(new Document { Data = context.ReadObject(result, "projection result") }); }
private DocumentItem ConvertRecordToDocumentItem(DocumentsOperationContext context, string[] csvReaderCurrentRecord, string[] csvReaderFieldHeaders, string collection) { try { var idStr = _hasId ? csvReaderCurrentRecord[_idIndex] : _hasCollection ? $"{csvReaderCurrentRecord[_collectionIndex]}/" : $"{collection}/"; var data = new DynamicJsonValue(); for (int i = 0; i < csvReaderFieldHeaders.Length; i++) { //ignoring reserved properties if (csvReaderFieldHeaders[i][0] == '@') { if (_hasCollection && i == _collectionIndex) { SetCollectionForDocument(csvReaderCurrentRecord[_collectionIndex], data); } continue; } if (_nestedPropertyDictionary != null && _nestedPropertyDictionary.TryGetValue(i, out var segments)) { var nestedData = data; for (var j = 0; ; j++) { //last segment holds the data if (j == segments.Length - 1) { nestedData[segments[j]] = ParseValue(csvReaderCurrentRecord[i]); break; //we are done } //Creating the objects along the path if needed e.g. Foo.Bar.Name will create the 'Bar' oject if needed if (nestedData[segments[j]] == null) { var tmpRef = new DynamicJsonValue(); nestedData[segments[j]] = tmpRef; nestedData = tmpRef; } //We need to advance into the nested object, since it is not the last segment it must be of type 'DynamicJsonValue' else { nestedData = (DynamicJsonValue)nestedData[segments[j]]; } } continue; } data[csvReaderFieldHeaders[i]] = ParseValue(csvReaderCurrentRecord[i]); } if (_hasCollection == false) { SetCollectionForDocument(collection, data); } return(new DocumentItem { Document = new Document { Data = context.ReadObject(data, idStr), Id = context.GetLazyString(idStr), ChangeVector = string.Empty, Flags = DocumentFlags.None, NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler, LastModified = _database.Time.GetUtcNow(), }, Attachments = null }); } finally { foreach (var disposeMe in _disposibales) { disposeMe.Dispose(); } _disposibales.Clear(); } }
public unsafe PatchResultData Apply(DocumentsOperationContext context, string documentKey, long?etag, PatchRequest patch, PatchRequest patchIfMissing, bool isTestOnly = false, bool skipPatchIfEtagMismatch = false) { var document = _database.DocumentsStorage.Get(context, documentKey); if (_logger.IsInfoEnabled) { _logger.Info(string.Format("Preparing to apply patch on ({0}). Document found?: {1}.", documentKey, document != null)); } if (etag.HasValue && document != null && document.Etag != etag.Value) { System.Diagnostics.Debug.Assert(document.Etag > 0); if (skipPatchIfEtagMismatch) { return(new PatchResultData { PatchResult = PatchResult.Skipped }); } if (_logger.IsInfoEnabled) { _logger.Info($"Got concurrent exception while tried to patch the following document: {documentKey}"); } throw new ConcurrencyException($"Could not patch document '{documentKey}' because non current etag was used") { ActualETag = document.Etag, ExpectedETag = etag.Value, }; } var patchRequest = patch; if (document == null) { if (patchIfMissing == null) { if (_logger.IsInfoEnabled) { _logger.Info("Tried to patch a not exists document and patchIfMissing is null"); } return(new PatchResultData { PatchResult = PatchResult.DocumentDoesNotExists }); } patchRequest = patchIfMissing; } var scope = ApplySingleScript(context, document, isTestOnly, patchRequest); var modifiedDocument = context.ReadObject(scope.ToBlittable(scope.PatchObject.AsObject()), documentKey, BlittableJsonDocumentBuilder.UsageMode.ToDisk); var result = new PatchResultData { PatchResult = PatchResult.NotModified, OriginalDocument = document?.Data, DebugInfo = scope.DebugInfo, }; if (modifiedDocument == null) { if (_logger.IsInfoEnabled) { _logger.Info($"After applying patch, modifiedDocument is null and document is null? {document == null}"); } result.PatchResult = PatchResult.Skipped; return(result); } if (isTestOnly) { return(new PatchResultData { PatchResult = PatchResult.Tested, OriginalDocument = document?.Data, ModifiedDocument = modifiedDocument, DebugActions = scope.DebugActions.GetDebugActions(), DebugInfo = scope.DebugInfo, }); } var putResult = new DocumentsStorage.PutOperationResults(); if (document == null) { putResult = _database.DocumentsStorage.Put(context, documentKey, null, modifiedDocument); } else { var isModified = document.Data.Size != modifiedDocument.Size; if (isModified == false) // optimization, if size different, no need to compute hash to check { var originHash = Hashing.XXHash64.Calculate(document.Data.BasePointer, (ulong)document.Data.Size); var modifiedHash = Hashing.XXHash64.Calculate(modifiedDocument.BasePointer, (ulong)modifiedDocument.Size); isModified = originHash != modifiedHash; } if (isModified) { putResult = _database.DocumentsStorage.Put(context, document.Key, document.Etag, modifiedDocument); result.PatchResult = PatchResult.Patched; } } if (putResult.Etag != 0) { result.Etag = putResult.Etag; result.Collection = putResult.Collection; } return(result); }
public void Execute(DocumentsOperationContext context, Transaction indexWriteTransaction) { // delete foreach (string reduceOutputId in _deletedReduceOutputs) { var referenceId = _index.OutputReduceToCollection.GetPatternGeneratedIdForReduceOutput(indexWriteTransaction, reduceOutputId); if (referenceId == null) { continue; } if (_idsToDeleteByReferenceDocumentId.TryGetValue(referenceId, out var values) == false) { values = new HashSet <string>(1, StringComparer.OrdinalIgnoreCase); _idsToDeleteByReferenceDocumentId.Add(referenceId, values); } values.Add(reduceOutputId); } foreach (var reduceReferenceIdToReduceOutputIds in _idsToDeleteByReferenceDocumentId) { using (var referenceDocument = _database.DocumentsStorage.Get(context, reduceReferenceIdToReduceOutputIds.Key)) { if (referenceDocument == null) { continue; } if (referenceDocument.Data.TryGet(nameof(OutputReduceToCollectionReference.ReduceOutputs), out BlittableJsonReaderArray ids) == false) { ThrowReduceOutputsPropertyNotFound(referenceDocument.Id); } var idsToRemove = reduceReferenceIdToReduceOutputIds.Value; if (idsToRemove.Count >= ids.Length) { Debug.Assert(ids.All(x => idsToRemove.Contains(x.ToString())), $"Found ID in {nameof(idsToRemove)} that aren't in {nameof(ids)}"); foreach (object deletedId in ids) { _index.OutputReduceToCollection.DeletePatternGeneratedIdForReduceOutput(indexWriteTransaction, deletedId.ToString()); } _database.DocumentsStorage.Delete(context, referenceDocument.Id, null); continue; } if (ids.Modifications == null) { ids.Modifications = new DynamicJsonArray(); } var indexesToRemove = new List <int>(); for (int i = ids.Length - 1; i >= 0; i--) { var id = ids[i].ToString(); if (idsToRemove.Contains(id)) { indexesToRemove.Add(i); } if (idsToRemove.Count == indexesToRemove.Count) { break; } } foreach (int toRemove in indexesToRemove) { ids.Modifications.RemoveAt(toRemove); } using (var doc = context.ReadObject(referenceDocument.Data, referenceDocument.Id)) { _database.DocumentsStorage.Put(context, referenceDocument.Id, null, doc); foreach (var idToRemove in idsToRemove) { _index.OutputReduceToCollection.DeletePatternGeneratedIdForReduceOutput(indexWriteTransaction, idToRemove); } } } } // put foreach (var referencesOfReduceOutput in _referencesOfReduceOutputs) { using (var existingReferenceDocument = _database.DocumentsStorage.Get(context, referencesOfReduceOutput.Key)) { var uniqueIds = referencesOfReduceOutput.Value; if (existingReferenceDocument != null) { if (existingReferenceDocument.Data.TryGet(nameof(OutputReduceToCollectionReference.ReduceOutputs), out BlittableJsonReaderArray existingIds) == false) { ThrowReduceOutputsPropertyNotFound(existingReferenceDocument.Id); } foreach (object id in existingIds) { uniqueIds.Add(id.ToString()); } } var referenceDoc = new DynamicJsonValue { [nameof(OutputReduceToCollectionReference.ReduceOutputs)] = new DynamicJsonArray(uniqueIds), [Constants.Documents.Metadata.Key] = new DynamicJsonValue { [Constants.Documents.Metadata.Collection] = _referencesCollectionName ?? $"{_outputReduceToCollection}/References" } }; using (var referenceJson = context.ReadObject(referenceDoc, "reference-of-reduce-output", BlittableJsonDocumentBuilder.UsageMode.ToDisk)) { _database.DocumentsStorage.Delete(context, referencesOfReduceOutput.Key, null); _database.DocumentsStorage.Put(context, referencesOfReduceOutput.Key, null, referenceJson, flags: DocumentFlags.Artificial | DocumentFlags.FromIndex); foreach (var reduceOutputId in referencesOfReduceOutput.Value) { _index.OutputReduceToCollection.AddPatternGeneratedIdForReduceOutput(indexWriteTransaction, reduceOutputId, referencesOfReduceOutput.Key); } } } } }
private void CleanupReferenceDocuments(DocumentsOperationContext context, Dictionary <string, HashSet <string> > idsToDeleteByReferenceDocumentId) { foreach (var reduceReferenceIdToReduceOutputIds in idsToDeleteByReferenceDocumentId) { using (var referenceDocument = _database.DocumentsStorage.Get(context, reduceReferenceIdToReduceOutputIds.Key)) { if (referenceDocument == null) { continue; } if (referenceDocument.Data.TryGet(nameof(OutputReduceToCollectionReference.ReduceOutputs), out BlittableJsonReaderArray ids) == false) { ThrowIdsPropertyNotFound(referenceDocument.Id); } var idsToRemove = reduceReferenceIdToReduceOutputIds.Value; if (ids.Modifications == null) { ids.Modifications = new DynamicJsonArray(); } var indexesToRemove = new List <int>(); for (int i = ids.Length - 1; i >= 0; i--) { var id = ids[i].ToString(); if (idsToRemove.Contains(id)) { indexesToRemove.Add(i); } if (idsToRemove.Count == indexesToRemove.Count) { break; } } foreach (int toRemove in indexesToRemove) { ids.Modifications.RemoveAt(toRemove); } using (var doc = context.ReadObject(referenceDocument.Data, referenceDocument.Id)) { if (doc.TryGet(nameof(OutputReduceToCollectionReference.ReduceOutputs), out BlittableJsonReaderArray updatedIds) == false) { ThrowIdsPropertyNotFound(referenceDocument.Id); } if (updatedIds.Length == 0) { _database.DocumentsStorage.Delete(context, referenceDocument.Id, null); } else { _database.DocumentsStorage.Put(context, referenceDocument.Id, null, doc); } } } } }
private async Task <(List <Match> Matches, GraphQueryPlan QueryPlan, bool NotModified)> GetQueryResults(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token, bool collectIntermediateResults = false) { var q = query.Metadata.Query; var qp = new GraphQueryPlan(query, documentsContext, existingResultEtag, token, Database) { CollectIntermediateResults = collectIntermediateResults }; qp.BuildQueryPlan(); qp.OptimizeQueryPlan(); //TODO: audit optimization if (query.WaitForNonStaleResults) { qp.IsStale = await qp.WaitForNonStaleResults(); } else { await qp.CreateAutoIndexesAndWaitIfNecessary(); } //for the case where we don't wait for non stale results we will override IsStale in the QueryQueryStep steps if (documentsContext.Transaction == null || documentsContext.Transaction.Disposed) { documentsContext.OpenReadTransaction(); } qp.ResultEtag = DocumentsStorage.ReadLastEtag(documentsContext.Transaction.InnerTransaction); if (existingResultEtag.HasValue) { if (qp.ResultEtag == existingResultEtag) { return(null, null, true); } } await qp.Initialize(); var matchResults = qp.Execute(); if (query.Metadata.OrderBy != null) { Sort(matchResults, query.Metadata.OrderBy, Database.Name, query.Query); } var filter = q.GraphQuery.Where; if (filter != null) { for (int i = 0; i < matchResults.Count; i++) { var resultAsJson = new DynamicJsonValue(); matchResults[i].PopulateVertices(resultAsJson); using (var result = documentsContext.ReadObject(resultAsJson, "graph/result")) { if (filter.IsMatchedBy(result, query.QueryParameters) == false) { matchResults[i] = default; } } } } if (query.Start > 0) { matchResults.RemoveRange(0, Math.Min(query.Start, matchResults.Count)); } if (query.PageSize < matchResults.Count) { matchResults.RemoveRange(query.PageSize, matchResults.Count - query.PageSize); } return(matchResults, qp, false); }
/// <summary> /// Iterates on a batch in document collection, process it and send documents if found any match /// </summary> /// <param name="docsContext"></param> /// <param name="sendingCurrentBatchStopwatch"></param> /// <returns>Whether succeeded finding any documents to send</returns> private async Task <bool> TrySendingBatchToClient(DocumentsOperationContext docsContext, Stopwatch sendingCurrentBatchStopwatch) { bool anyDocumentsSentInCurrentIteration = false; int docsToFlush = 0; using (var writer = new BlittableJsonTextWriter(docsContext, _buffer)) { using (docsContext.OpenReadTransaction()) { foreach (var result in _documentsFetcher.GetDataToSend(docsContext, _startEtag)) { _startEtag = result.Doc.Etag; _lastChangeVector = string.IsNullOrEmpty(SubscriptionState.ChangeVectorForNextBatchStartingPoint) ? result.Doc.ChangeVector : ChangeVectorUtils.MergeVectors(result.Doc.ChangeVector, SubscriptionState.ChangeVectorForNextBatchStartingPoint); if (result.Doc.Data == null) { if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000) { await SendHeartBeat(); sendingCurrentBatchStopwatch.Restart(); } continue; } anyDocumentsSentInCurrentIteration = true; writer.WriteStartObject(); writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(TypeSegment)); writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(DataSegment)); writer.WriteComma(); writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(DataSegment)); result.Doc.EnsureMetadata(); if (result.Exception != null) { var metadata = result.Doc.Data[Client.Constants.Documents.Metadata.Key]; writer.WriteValue(BlittableJsonToken.StartObject, docsContext.ReadObject(new DynamicJsonValue { [Client.Constants.Documents.Metadata.Key] = metadata }, result.Doc.Id) ); writer.WriteComma(); writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(ExceptionSegment)); writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(result.Exception.ToString())); } else { writer.WriteDocument(docsContext, result.Doc, metadataOnly: false); } writer.WriteEndObject(); docsToFlush++; // perform flush for current batch after 1000ms of running or 1 MB if (_buffer.Length > Constants.Size.Megabyte || sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000) { if (docsToFlush > 0) { await FlushDocsToClient(writer, docsToFlush); docsToFlush = 0; sendingCurrentBatchStopwatch.Restart(); } else { await SendHeartBeat(); } } } } if (anyDocumentsSentInCurrentIteration) { docsContext.Write(writer, new DynamicJsonValue { [nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.EndOfBatch) }); await FlushDocsToClient(writer, docsToFlush, true); if (_logger.IsInfoEnabled) { _logger.Info( $"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}"); } } } return(anyDocumentsSentInCurrentIteration); }
public void Put(DocumentsOperationContext context, string id, BlittableJsonReaderObject document, DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags, string changeVector, long lastModifiedTicks, RevisionsCollectionConfiguration configuration = null, CollectionName collectionName = null) { Debug.Assert(changeVector != null, "Change vector must be set"); Debug.Assert(lastModifiedTicks != DateTime.MinValue.Ticks, "last modified ticks must be set"); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); if (collectionName == null) { collectionName = _database.DocumentsStorage.ExtractCollectionName(context, document); } if (configuration == null) { configuration = GetRevisionsConfiguration(collectionName.Name); } using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { var fromReplication = (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) == NonPersistentDocumentFlags.FromReplication; var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName); // We want the revision's attachments to have a lower etag than the revision itself if ((flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments) { if (flags.Contain(DocumentFlags.Revision) == false) { using (Slice.From(context.Allocator, changeVector, out Slice changeVectorSlice)) { if (table.VerifyKeyExists(changeVectorSlice) == false) { _documentsStorage.AttachmentsStorage.RevisionAttachments(context, lowerId, changeVectorSlice); } } } } if (document.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) && metadata.TryGet(Constants.Documents.Metadata.Counters, out BlittableJsonReaderArray counterNames)) { var dvj = new DynamicJsonValue(); for (var i = 0; i < counterNames.Length; i++) { var counter = counterNames[i].ToString(); var val = _documentsStorage.CountersStorage.GetCounterValue(context, id, counter); if (val == null) { continue; } dvj[counter] = val.Value; } metadata.Modifications = new DynamicJsonValue(metadata) { [Constants.Documents.Metadata.RevisionCounters] = dvj }; metadata.Modifications.Remove(Constants.Documents.Metadata.Counters); document.Modifications = new DynamicJsonValue(document) { [Constants.Documents.Metadata.Key] = metadata }; document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); } if (fromReplication) { void PutFromRevisionIfChangeVectorIsGreater() { bool hasDoc; TableValueReader tvr; try { hasDoc = _documentsStorage.GetTableValueReaderForDocument(context, lowerId, throwOnConflict: true, tvr: out tvr); } catch (DocumentConflictException) { // Do not modify the document. return; } if (hasDoc == false) { PutFromRevision(); return; } var docChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref tvr); if (ChangeVectorUtils.GetConflictStatus(changeVector, docChangeVector) == ConflictStatus.Update) { PutFromRevision(); } void PutFromRevision() { _documentsStorage.Put(context, id, null, document, lastModifiedTicks, changeVector, flags & ~DocumentFlags.Revision, nonPersistentFlags | NonPersistentDocumentFlags.FromRevision); } } PutFromRevisionIfChangeVectorIsGreater(); } flags |= DocumentFlags.Revision; var newEtag = _database.DocumentsStorage.GenerateNextEtag(); var newEtagSwapBytes = Bits.SwapBytes(newEtag); using (table.Allocate(out TableValueBuilder tvb)) using (Slice.From(context.Allocator, changeVector, out var cv)) { tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(lowerId); tvb.Add(SpecialChars.RecordSeparator); tvb.Add(newEtagSwapBytes); tvb.Add(idPtr); tvb.Add(document.BasePointer, document.Size); tvb.Add((int)flags); tvb.Add(NotDeletedRevisionMarker); tvb.Add(lastModifiedTicks); tvb.Add(context.GetTransactionMarker()); if (flags.Contain(DocumentFlags.Resolved)) { tvb.Add((int)DocumentFlags.Resolved); } else { tvb.Add(0); } tvb.Add(Bits.SwapBytes(lastModifiedTicks)); var isNew = table.Set(tvb); if (isNew == false) { // It might be just an update from replication as we call this twice, both for the doc delete and for deleteRevision. return; } } DeleteOldRevisions(context, table, lowerId, collectionName, configuration, nonPersistentFlags, changeVector, lastModifiedTicks); } }
public override void Execute(DocumentsOperationContext context, RavenTransaction tx) { var hiLoDocumentKey = RavenKeyGeneratorsHilo + Key; var prefix = Key + Separator; long oldMax = 0; var newDoc = new DynamicJsonValue(); BlittableJsonReaderObject hiloDocReader = null, serverPrefixDocReader = null; try { try { serverPrefixDocReader = Database.DocumentsStorage.Get(context, RavenKeyServerPrefix)?.Data; hiloDocReader = Database.DocumentsStorage.Get(context, hiLoDocumentKey)?.Data; } catch (DocumentConflictException e) { throw new InvalidDataException(@"Failed to fetch HiLo document due to a conflict on the document. This shouldn't happen, since it this conflict should've been resolved during replication. This exception should not happen and is likely a bug.", e); } string serverPrefix; if (serverPrefixDocReader != null && serverPrefixDocReader.TryGet("ServerPrefix", out serverPrefix)) { prefix += serverPrefix; } if (hiloDocReader != null) { hiloDocReader.TryGet("Max", out oldMax); var prop = new BlittableJsonReaderObject.PropertyDetails(); for (var i = 0; i < hiloDocReader.Count; i++) { hiloDocReader.GetPropertyByIndex(0, ref prop); if (prop.Name == "Max") { continue; } newDoc[prop.Name] = prop.Value; } } } finally { serverPrefixDocReader?.Dispose(); hiloDocReader?.Dispose(); } oldMax = Math.Max(oldMax, LastRangeMax); newDoc["Max"] = oldMax + Capacity; using ( var freshHilo = context.ReadObject(newDoc, hiLoDocumentKey, BlittableJsonDocumentBuilder.UsageMode.ToDisk)) { Database.DocumentsStorage.Put(context, hiLoDocumentKey, null, freshHilo); } OldMax = oldMax; Prefix = prefix; }