Exemplo n.º 1
0
        internal static BlittableJsonReaderObject EnsureLowerCasedIndexIdProperty(DocumentsOperationContext context, BlittableJsonReaderObject json,
                                                                                  ElasticSearchIndexWithRecords index)
        {
            if (json.TryGet(index.DocumentIdProperty, out LazyStringValue idProperty))
            {
                using (var old = json)
                {
                    json.Modifications = new DynamicJsonValue(json)
                    {
                        [index.DocumentIdProperty] = LowerCaseDocumentIdProperty(idProperty)
                    };

                    json = context.ReadObject(json, "es-etl-load");
                }
            }
            else if (json.Modifications != null)
            {
                // document id property was not added by user, so we inserted the lowercased id in ElasticSearchDocumentTransformer.LoadToFunction
#if DEBUG
                var docIdProperty = json.Modifications.Properties.First(x => x.Name == index.DocumentIdProperty);

                Debug.Assert(docIdProperty.Value.ToString() == docIdProperty.Value.ToString().ToLowerInvariant());
#endif

                json = context.ReadObject(json, "es-etl-load");
            }

            return(json);
        }
Exemplo n.º 2
0
            protected override long ExecuteCmd(DocumentsOperationContext context)
            {
                var hiLoDocumentId = RavenHiloIdPrefix + Key;
                var prefix         = Key + Separator;

                var newDoc = new DynamicJsonValue();
                BlittableJsonReaderObject hiloDocReader = null;

                try
                {
                    try
                    {
                        hiloDocReader = Database.DocumentsStorage.Get(context, hiLoDocumentId)?.Data;
                    }
                    catch (DocumentConflictException e)
                    {
                        throw new InvalidDataException("Failed to fetch HiLo document due to a conflict on the document. " +
                                                       "This shouldn't happen, since it this conflict should've been resolved during replication. " +
                                                       "This exception should not happen and is likely a bug.", e);
                    }

                    if (hiloDocReader == null)
                    {
                        OldMax        = LastRangeMax;
                        newDoc["Max"] = OldMax + Capacity;
                        newDoc[Constants.Documents.Metadata.Key] = new DynamicJsonValue
                        {
                            [Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection
                        };

                        using (var freshHilo = context.ReadObject(newDoc, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk))
                            Database.DocumentsStorage.Put(context, hiLoDocumentId, null, freshHilo);
                    }
                    else
                    {
                        hiloDocReader.TryGet("Max", out long oldMax);
                        OldMax = Math.Max(oldMax, LastRangeMax);

                        hiloDocReader.Modifications = new DynamicJsonValue(hiloDocReader)
                        {
                            ["Max"] = OldMax + Capacity
                        };

                        using (var freshHilo = context.ReadObject(hiloDocReader, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk))
                            Database.DocumentsStorage.Put(context, hiLoDocumentId, null, freshHilo);
                    }

                    Prefix = prefix;
                }
                finally
                {
                    hiloDocReader?.Dispose();
                }
                return(1);
            }
Exemplo n.º 3
0
            protected override long ExecuteCmd(DocumentsOperationContext context)
            {
                var hiLoDocumentId = RavenHiloIdPrefix + Key;

                var document = Database.DocumentsStorage.Get(context, hiLoDocumentId);

                if (document == null)
                {
                    return(1);
                }

                document.Data.TryGet("Max", out long oldMax);
                if (oldMax != End || Last > oldMax)
                {
                    return(1);
                }

                document.Data.Modifications = new DynamicJsonValue
                {
                    ["Max"] = Last
                };

                using (var hiloReader = context.ReadObject(document.Data, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk))
                {
                    Database.DocumentsStorage.Put(context, hiLoDocumentId, null, hiloReader);
                }

                return(1);
            }
Exemplo n.º 4
0
        public static LegacyAttachmentDetails GenerateLegacyAttachmentDetails(
            DocumentsOperationContext context,
            Stream decodedStream,
            string key,
            BlittableJsonReaderObject metadata,
            ref DocumentItem.AttachmentStream attachment)
        {
            var stream = attachment.Stream;
            var hash   = AsyncHelpers.RunSync(() => AttachmentsStorageHelper.CopyStreamToFileAndCalculateHash(context, decodedStream, stream, CancellationToken.None));

            attachment.Stream.Flush();
            var lazyHash = context.GetLazyString(hash);

            attachment.Base64HashDispose = Slice.External(context.Allocator, lazyHash, out attachment.Base64Hash);
            var tag     = $"{DummyDocumentPrefix}{key}{RecordSeparator}d{RecordSeparator}{key}{RecordSeparator}{hash}{RecordSeparator}";
            var lazyTag = context.GetLazyString(tag);

            attachment.TagDispose = Slice.External(context.Allocator, lazyTag, out attachment.Tag);
            var id     = GetLegacyAttachmentId(key);
            var lazyId = context.GetLazyString(id);

            attachment.Data = context.ReadObject(metadata, id);
            return(new LegacyAttachmentDetails
            {
                Id = lazyId,
                Hash = hash,
                Key = key,
                Size = attachment.Stream.Length,
                Tag = tag,
                Metadata = attachment.Data
            });
        }
Exemplo n.º 5
0
        public static BlittableJsonReaderObject WriteDummyDocumentForAttachment(DocumentsOperationContext context, LegacyAttachmentDetails details)
        {
            var attachment = new DynamicJsonValue
            {
                ["Name"]        = details.Key,
                ["Hash"]        = details.Hash,
                ["ContentType"] = string.Empty,
                ["Size"]        = details.Size,
            };
            var attachments = new DynamicJsonArray();

            attachments.Add(attachment);
            var metadata = new DynamicJsonValue
            {
                [Constants.Documents.Metadata.Collection]  = "@files",
                [Constants.Documents.Metadata.Attachments] = attachments,
                [Constants.Documents.Metadata.LegacyAttachmentsMetadata] = details.Metadata
            };
            var djv = new DynamicJsonValue
            {
                [Constants.Documents.Metadata.Key] = metadata,
            };

            return(context.ReadObject(djv, details.Id));
        }
Exemplo n.º 6
0
        private static void HandleResultsWithoutSelect <TResult>(
            DocumentsOperationContext documentsContext,
            List <Match> matchResults, TResult final) where TResult : QueryResultServerSide <Document>
        {
            foreach (var match in matchResults)
            {
                if (match.Empty)
                {
                    continue;
                }

                if (match.Count == 1) //if we don't have multiple results in each row, we can "flatten" the row
                {
                    final.AddResult(match.GetFirstResult());
                    continue;
                }

                var resultAsJson = new DynamicJsonValue();
                match.PopulateVertices(resultAsJson);

                var result = new Document
                {
                    Data = documentsContext.ReadObject(resultAsJson, "graph/result"),
                };

                final.AddResult(result);
            }
        }
Exemplo n.º 7
0
        private void WriteAllEnvs(BlittableJsonTextWriter writer, DocumentsOperationContext context)
        {
            var envs = Database.GetAllStoragesEnvironment();

            bool first = true;

            foreach (var env in envs)
            {
                if (env == null)
                {
                    continue;
                }

                if (!first)
                {
                    writer.WriteComma();
                }
                first = false;

                writer.WriteStartObject();
                writer.WritePropertyName("Environment");
                writer.WriteString(env.Name);
                writer.WriteComma();

                writer.WritePropertyName("Type");
                writer.WriteString(env.Type.ToString());
                writer.WriteComma();

                var djv = (DynamicJsonValue)TypeConverter.ToBlittableSupportedType(GetDetailedReport(env, false));
                writer.WritePropertyName("Report");
                writer.WriteObject(context.ReadObject(djv, env.Name));

                writer.WriteEndObject();
            }
        }
Exemplo n.º 8
0
            public override void Execute(DocumentsOperationContext context, RavenTransaction tx)
            {
                var hiLoDocumentKey = RavenKeyGeneratorsHilo + Key;

                var document = Database.DocumentsStorage.Get(context, hiLoDocumentKey);

                if (document == null)
                {
                    return;
                }

                long oldMax;

                document.Data.TryGet("Max", out oldMax);

                if (oldMax != End || Last > oldMax)
                {
                    return;
                }

                document.Data.Modifications = new DynamicJsonValue()
                {
                    ["Max"] = Last,
                };

                using (var hiloReader = context.ReadObject(document.Data, hiLoDocumentKey, BlittableJsonDocumentBuilder.UsageMode.ToDisk))
                {
                    Database.DocumentsStorage.Put(context, hiLoDocumentKey, null, hiloReader);
                }
            }
Exemplo n.º 9
0
        public void AddIncludesToResult(TimeSeriesRangeResult rangeResult)
        {
            if (rangeResult == null || _includes?.Properties.Count > 0 == false)
            {
                return;
            }

            rangeResult.Includes = _context.ReadObject(_includes, "TimeSeriesRangeIncludes/" + _docId);
        }
Exemplo n.º 10
0
            public override int Execute(DocumentsOperationContext context)
            {
                var hiLoDocumentId = RavenIdGeneratorsHilo + Key;
                var prefix         = Key + Separator;

                long oldMax = 0;
                var  newDoc = new DynamicJsonValue();
                BlittableJsonReaderObject hiloDocReader = null;

                try
                {
                    try
                    {
                        hiloDocReader = Database.DocumentsStorage.Get(context, hiLoDocumentId)?.Data;
                    }
                    catch (DocumentConflictException e)
                    {
                        throw new InvalidDataException("Failed to fetch HiLo document due to a conflict on the document. " +
                                                       "This shouldn't happen, since it this conflict should've been resolved during replication. " +
                                                       "This exception should not happen and is likely a bug.", e);
                    }

                    if (hiloDocReader != null)
                    {
                        var prop = new BlittableJsonReaderObject.PropertyDetails();
                        foreach (var propertyId in hiloDocReader.GetPropertiesByInsertionOrder())
                        {
                            hiloDocReader.GetPropertyByIndex(propertyId, ref prop);
                            if (prop.Name == "Max")
                            {
                                oldMax = (long)prop.Value;
                                continue;
                            }

                            newDoc[prop.Name] = prop.Value;
                        }
                    }

                    oldMax = Math.Max(oldMax, LastRangeMax);

                    newDoc["Max"] = oldMax + Capacity;

                    using (var freshHilo = context.ReadObject(newDoc, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk))
                        Database.DocumentsStorage.Put(context, hiLoDocumentId, null, freshHilo);

                    OldMax = oldMax;
                    Prefix = prefix;
                }
                finally
                {
                    hiloDocReader?.Dispose();
                }
                return(1);
            }
Exemplo n.º 11
0
        private void WriteLastEtag(DocumentsOperationContext context)
        {
            var key      = Constants.SqlReplication.RavenSqlReplicationStatusPrefix + ReplicationUniqueName;
            var document = context.ReadObject(new DynamicJsonValue
            {
                ["Name"] = ReplicationUniqueName,
                ["LastReplicatedEtag"] = Statistics.LastReplicatedEtag,
                ["LastTombstonesEtag"] = Statistics.LastTombstonesEtag,
            }, key, BlittableJsonDocumentBuilder.UsageMode.ToDisk);

            _database.DocumentsStorage.Put(context, key, null, document);
        }
Exemplo n.º 12
0
 public void Delete(DocumentsOperationContext context, string id, Slice lowerId, CollectionName collectionName, string changeVector,
                    long lastModifiedTicks, NonPersistentDocumentFlags nonPersistentFlags, DocumentFlags flags)
 {
     using (DocumentIdWorker.GetStringPreserveCase(context, id, out Slice idPtr))
     {
         var deleteRevisionDocument = context.ReadObject(new DynamicJsonValue
         {
             [Constants.Documents.Metadata.Key] = new DynamicJsonValue
             {
                 [Constants.Documents.Metadata.Collection] = collectionName.Name
             }
         }, "RevisionsBin");
         Delete(context, lowerId, idPtr, id, collectionName, deleteRevisionDocument, changeVector, lastModifiedTicks, nonPersistentFlags, flags);
     }
 }
Exemplo n.º 13
0
 private BlittableJsonReaderObject GetCleanMetadata(BlittableJsonReaderObject metadata, DocumentsOperationContext context)
 {
     metadata.Modifications = new DynamicJsonValue(metadata);
     metadata.Modifications.Remove("Origin");
     metadata.Modifications.Remove("Raven-Synchronization-Version");
     metadata.Modifications.Remove("Raven-Synchronization-Source");
     metadata.Modifications.Remove("Creation-Date");
     metadata.Modifications.Remove("Raven-Creation-Date");
     metadata.Modifications.Remove("Raven-Synchronization-History");
     metadata.Modifications.Remove("RavenFS-Size");
     metadata.Modifications.Remove("Last-Modified");
     metadata.Modifications.Remove("Raven-Last-Modified");
     metadata.Modifications.Remove("Content-MD5");
     metadata.Modifications.Remove("ETag");
     return(context.ReadObject(metadata, MigrationStateKey));
 }
Exemplo n.º 14
0
        public virtual string PutDocument(string key, JsValue document, JsValue metadata, JsValue etagJs, Engine engine)
        {
            if (document.IsObject() == false)
            {
                throw new InvalidOperationException(
                          $"Created document must be a valid object which is not null or empty. Document key: '{key}'.");
            }

            long?etag = null;

            if (etagJs.IsNumber())
            {
                etag = (long)etagJs.AsNumber();
            }
            else if (etagJs.IsNull() == false && etagJs.IsUndefined() == false && etagJs.ToString() != "None")
            {
                throw new InvalidOperationException($"Invalid ETag value for document '{key}'");
            }

            var data = ToBlittable(document.AsObject());

            if (metadata.IsObject())
            {
                data["@metadata"] = ToBlittable(metadata.AsObject());
            }

            if (DebugMode)
            {
                DebugActions.PutDocument.Add(new DynamicJsonValue
                {
                    ["Key"]  = key,
                    ["Etag"] = etag,
                    ["Data"] = data,
                });
            }

            var dataReader = _context.ReadObject(data, key, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
            var put        = _database.DocumentsStorage.Put(_context, key, etag, dataReader);

            return(put.Key);
        }
        private void WriteIntermidiateResults(List <GraphQueryRunner.Match> matches)
        {
            _writer.WritePropertyName("Results");
            _writer.WriteStartArray();
            var first = true;

            foreach (var match in matches)
            {
                if (first == false)
                {
                    _writer.WriteComma();
                }

                first = false;
                var djv = new DynamicJsonValue();
                match.PopulateVertices(djv);
                _writer.WriteObject(_ctx.ReadObject(djv, null));
            }

            _writer.WriteEndArray();
        }
Exemplo n.º 16
0
        public virtual PatchResultData Apply(DocumentsOperationContext context, Document document, PatchRequest patch)
        {
            if (document == null)
            {
                return(null);
            }

            if (string.IsNullOrEmpty(patch.Script))
            {
                throw new InvalidOperationException("Patch script must be non-null and not empty");
            }

            var scope            = ApplySingleScript(context, document, false, patch);
            var modifiedDocument = context.ReadObject(scope.ToBlittable(scope.PatchObject.AsObject()), document.Key); /* TODO: Should not use BlittableJsonDocumentBuilder.UsageMode.ToDisk? */

            return(new PatchResultData
            {
                ModifiedDocument = modifiedDocument ?? document.Data,
                DebugInfo = scope.DebugInfo,
            });
        }
Exemplo n.º 17
0
            private JsValue GetMetadata(JsValue self, JsValue[] args)
            {
                if (args.Length != 1 || !(args[0].AsObject() is BlittableObjectInstance boi))
                {
                    throw new InvalidOperationException("getMetadata(doc) must be called with a single entity argument");
                }

                if (!(boi.Blittable[Constants.Documents.Metadata.Key] is BlittableJsonReaderObject metadata))
                {
                    return(JsValue.Null);
                }

                metadata.Modifications = new DynamicJsonValue
                {
                    [Constants.Documents.Metadata.ChangeVector] = boi.ChangeVector,
                    [Constants.Documents.Metadata.Id]           = boi.DocumentId,
                    [Constants.Documents.Metadata.LastModified] = boi.LastModified,
                };

                metadata = _context.ReadObject(metadata, boi.DocumentId);

                return(TranslateToJs(ScriptEngine, _context, metadata));
            }
Exemplo n.º 18
0
        public void Put(DocumentsOperationContext context, string id, BlittableJsonReaderObject document,
                        DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags, string changeVector, long lastModifiedTicks,
                        RevisionsCollectionConfiguration configuration = null, CollectionName collectionName = null)
        {
            Debug.Assert(changeVector != null, "Change vector must be set");

            BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);

            if (collectionName == null)
            {
                collectionName = _database.DocumentsStorage.ExtractCollectionName(context, id, document);
            }

            using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
            {
                var fromSmuggler    = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler;
                var fromReplication = (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) == NonPersistentDocumentFlags.FromReplication;

                var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName);

                // We want the revision's attachments to have a lower etag than the revision itself
                if ((flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments &&
                    fromSmuggler == false)
                {
                    using (Slice.From(context.Allocator, changeVector, out Slice changeVectorSlice))
                    {
                        if (table.VerifyKeyExists(changeVectorSlice) == false)
                        {
                            _documentsStorage.AttachmentsStorage.RevisionAttachments(context, lowerId, changeVectorSlice);
                        }
                    }
                }

                if (fromReplication)
                {
                    void PutFromRevisionIfChangeVectorIsGreater()
                    {
                        bool             hasDoc;
                        TableValueReader tvr;

                        try
                        {
                            hasDoc = _documentsStorage.GetTableValueReaderForDocument(context, lowerId, throwOnConflict: true, tvr: out tvr);
                        }
                        catch (DocumentConflictException)
                        {
                            // Do not modify the document.
                            return;
                        }

                        if (hasDoc == false)
                        {
                            PutFromRevision();
                            return;
                        }

                        var docChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref tvr);

                        if (ChangeVectorUtils.GetConflictStatus(changeVector, docChangeVector) == ConflictStatus.Update)
                        {
                            PutFromRevision();
                        }

                        void PutFromRevision()
                        {
                            _documentsStorage.Put(context, id, null, document, lastModifiedTicks, changeVector,
                                                  flags & ~DocumentFlags.Revision, nonPersistentFlags | NonPersistentDocumentFlags.FromRevision);
                        }
                    }

                    PutFromRevisionIfChangeVectorIsGreater();
                }

                flags |= DocumentFlags.Revision;
                var data             = context.ReadObject(document, id);
                var newEtag          = _database.DocumentsStorage.GenerateNextEtag();
                var newEtagSwapBytes = Bits.SwapBytes(newEtag);

                using (table.Allocate(out TableValueBuilder tvb))
                    using (Slice.From(context.Allocator, changeVector, out var cv))
                    {
                        tvb.Add(cv.Content.Ptr, cv.Size);
                        tvb.Add(lowerId);
                        tvb.Add(SpecialChars.RecordSeparator);
                        tvb.Add(newEtagSwapBytes);
                        tvb.Add(idPtr);
                        tvb.Add(data.BasePointer, data.Size);
                        tvb.Add((int)flags);
                        tvb.Add(newEtagSwapBytes);
                        tvb.Add(lastModifiedTicks);
                        tvb.Add(context.GetTransactionMarker());
                        var isNew = table.Set(tvb);
                        if (isNew == false)
                        {
                            // It might be just an update from replication as we call this twice, both for the doc delete and for deleteRevision.
                            return;
                        }
                    }

                if (configuration == null)
                {
                    configuration = GetRevisionsConfiguration(collectionName.Name);
                }

                DeleteOldRevisions(context, table, lowerId, collectionName, configuration, nonPersistentFlags, changeVector);
            }
        }
Exemplo n.º 19
0
        private void DeleteCounter(UpdateStep step, LazyStringValue tombstoneKey, DocumentsOperationContext context)
        {
            var(docId, counterName) = ExtractDocIdAndNameFromCounterTombstone(context, tombstoneKey);

            using (docId)
                using (counterName)
                    using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, docId, out Slice lowerId, out _))
                    {
                        string collection = null;

                        var docsTable = new Table(DocsSchema, step.ReadTx);
                        if (docsTable.ReadByKey(lowerId, out var tvr))
                        {
                            using (var doc = new BlittableJsonReaderObject(tvr.Read((int)DocumentsTable.Data, out int size), size, context))
                            {
                                collection = CollectionName.GetCollectionName(doc);
                            }
                        }

                        var collectionName = new CollectionName(collection);
                        var table          = step.DocumentsStorage.CountersStorage.GetCountersTable(step.WriteTx, collectionName);

                        if (table.ReadByKey(lowerId, out var existing) == false)
                        {
                            return;
                        }

                        // (int)CountersTable.Data = 3
                        var data = new BlittableJsonReaderObject(existing.Read(3, out int oldSize), oldSize, context);

                        if (data.TryGet(CountersStorage.Values, out BlittableJsonReaderObject counters) == false ||
                            counters.TryGetMember(counterName, out object counterToDelete) == false ||
                            counterToDelete is LazyStringValue) // already deleted
                        {
                            return;
                        }

                        var deleteCv = step.DocumentsStorage.CountersStorage.GenerateDeleteChangeVectorFromRawBlob(data, counterToDelete as BlittableJsonReaderObject.RawBlob);
                        counters.Modifications = new DynamicJsonValue(counters)
                        {
                            [counterName] = deleteCv
                        };

                        using (var old = data)
                        {
                            data = context.ReadObject(data, null, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
                        }

                        var newEtag         = step.DocumentsStorage.GenerateNextEtag();
                        var newChangeVector = ChangeVectorUtils.NewChangeVector(step.DocumentsStorage.DocumentDatabase.ServerStore.NodeTag, newEtag, _dbId);
                        using (data)
                            using (Slice.From(context.Allocator, newChangeVector, out var cv))
                                using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice))
                                    using (table.Allocate(out TableValueBuilder tvb))
                                    {
                                        tvb.Add(lowerId);
                                        tvb.Add(Bits.SwapBytes(newEtag));
                                        tvb.Add(cv);
                                        tvb.Add(data.BasePointer, data.Size);
                                        tvb.Add(collectionSlice);
                                        tvb.Add(context.GetTransactionMarker());

                                        table.Set(tvb);
                                    }
                    }
        }
Exemplo n.º 20
0
        public PutOperationResults PutDocument(DocumentsOperationContext context, string id,
                                               string expectedChangeVector,
                                               BlittableJsonReaderObject document,
                                               long?lastModifiedTicks = null,
                                               string changeVector    = null,
                                               DocumentFlags flags    = DocumentFlags.None,
                                               NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None)
        {
            if (context.Transaction == null)
            {
                ThrowRequiresTransaction();
                return(default(PutOperationResults)); // never hit
            }

#if DEBUG
            var documentDebugHash = document.DebugHash;
            document.BlittableValidation();
            BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
            AssertMetadataWasFiltered(document);
#endif

            var newEtag       = _documentsStorage.GenerateNextEtag();
            var modifiedTicks = lastModifiedTicks ?? _documentDatabase.Time.GetUtcNow().Ticks;

            id = BuildDocumentId(id, newEtag, out bool knownNewId);
            using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
            {
                var collectionName = _documentsStorage.ExtractCollectionName(context, document);
                var table          = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));

                var oldValue = default(TableValueReader);
                if (knownNewId == false)
                {
                    // delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it
                    DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size);

                    table.ReadByKey(lowerId, out oldValue);
                }

                BlittableJsonReaderObject oldDoc = null;
                if (oldValue.Pointer == null)
                {
                    // expectedChangeVector being null means we don't care, and empty
                    // means that it must be new
                    if (string.IsNullOrEmpty(expectedChangeVector) == false)
                    {
                        ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector);
                    }
                }
                else
                {
                    // expectedChangeVector  has special meaning here
                    // null - means, don't care, don't check
                    // "" / empty - means, must be new
                    // anything else - must match exactly
                    if (expectedChangeVector != null)
                    {
                        var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue);
                        if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0)
                        {
                            ThrowConcurrentException(id, expectedChangeVector, oldChangeVector);
                        }
                    }

                    oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context);
                    var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc);
                    if (oldCollectionName != collectionName)
                    {
                        ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName);
                    }

                    var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue);

                    if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate &&
                        (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication)
                    {
                        if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
                        {
                            flags |= DocumentFlags.HasAttachments;
                        }
                    }
                }

                var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue);
                changeVector        = result.ChangeVector;
                nonPersistentFlags |= result.NonPersistentFlags;
                if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved))
                {
                    flags |= DocumentFlags.Resolved;
                }

                if (collectionName.IsHiLo == false &&
                    (flags & DocumentFlags.Artificial) != DocumentFlags.Artificial)
                {
                    if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags))
                    {
#if DEBUG
                        if (document.DebugHash != documentDebugHash)
                        {
                            throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
                                                           "this is likely because you are trying to save a document that is already stored and was moved");
                        }
#endif
                        document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
#if DEBUG
                        documentDebugHash = document.DebugHash;
                        document.BlittableValidation();
                        BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
                        AssertMetadataWasFiltered(document);
                        AttachmentsStorage.AssertAttachments(document, flags);
#endif
                    }

                    if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false &&
                        (flags.Contain(DocumentFlags.Resolved) ||
                         _documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null
                        ))
                    {
                        var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document,
                                                                                                                      ref flags, out RevisionsCollectionConfiguration configuration);
                        if (shouldVersion)
                        {
                            _documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags,
                                                                                    changeVector, modifiedTicks, configuration, collectionName);
                        }
                    }
                }

                using (Slice.From(context.Allocator, changeVector, out var cv))
                    using (table.Allocate(out TableValueBuilder tvb))
                    {
                        tvb.Add(lowerId);
                        tvb.Add(Bits.SwapBytes(newEtag));
                        tvb.Add(idPtr);
                        tvb.Add(document.BasePointer, document.Size);
                        tvb.Add(cv.Content.Ptr, cv.Size);
                        tvb.Add(modifiedTicks);
                        tvb.Add((int)flags);
                        tvb.Add(context.GetTransactionMarker());

                        if (oldValue.Pointer == null)
                        {
                            table.Insert(tvb);
                        }
                        else
                        {
                            table.Update(oldValue.Id, tvb);
                        }
                    }

                if (collectionName.IsHiLo == false)
                {
                    _documentsStorage.ExpirationStorage.Put(context, lowerId, document);
                }

                context.LastDatabaseChangeVector = changeVector;
                _documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1);
                _documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size);

                context.Transaction.AddAfterCommitNotification(new DocumentChange
                {
                    ChangeVector   = changeVector,
                    CollectionName = collectionName.Name,
                    Id             = id,
                    Type           = DocumentChangeTypes.Put,
                });

#if DEBUG
                if (document.DebugHash != documentDebugHash)
                {
                    throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
                                                   "this is likely because you are trying to save a document that is already stored and was moved");
                }
                document.BlittableValidation();
                BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
                AssertMetadataWasFiltered(document);
                AttachmentsStorage.AssertAttachments(document, flags);
#endif
                return(new PutOperationResults
                {
                    Etag = newEtag,
                    Id = id,
                    Collection = collectionName,
                    ChangeVector = changeVector,
                    Flags = flags,
                    LastModified = new DateTime(modifiedTicks)
                });
            }
        }
Exemplo n.º 21
0
        internal Document ProjectFromMatch(Match match, JsonOperationContext context, CancellationToken token)
        {
            var result = new DynamicJsonValue();

            result[Constants.Documents.Metadata.Key] = new DynamicJsonValue
            {
                [Constants.Documents.Metadata.Projection] = true
            };

            var item = new Document();

            foreach (var fieldToFetch in FieldsToFetch.Fields.Values)
            {
                object fieldVal;
                string key;
                if (fieldToFetch.QueryField?.ExpressionField?.Compound?.Contains("[]") ?? false)
                {
                    key = fieldToFetch.QueryField.ExpressionField.Compound.LastOrDefault().Value;
                }
                else
                {
                    key = fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value;
                }

                if (fieldToFetch.QueryField.Function != null)
                {
                    var args = new object[fieldToFetch.FunctionArgs.Length + 1];
                    for (int i = 0; i < fieldToFetch.FunctionArgs.Length; i++)
                    {
                        var val = match.GetResult(fieldToFetch.FunctionArgs[i].ProjectedName);
                        switch (val)
                        {
                        case Document doc:
                            doc.EnsureMetadata();
                            args[i] = doc;
                            break;

                        case BlittableJsonReaderObject bjro:
                            args[i] = bjro;
                            break;

                        case List <Match> matchList:
                            CreateArgsFromMatchList(matchList, args, i);
                            break;

                        case MatchCollection matchCollection:
                            CreateArgsFromMatchList(matchCollection, args, i);
                            break;

                        case string s:
                            args[i] = s;
                            break;

                        default:
                            args[i] = null;
                            break;
                        }
                    }

                    key      = fieldToFetch.ProjectedName ?? (fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value);
                    fieldVal = GetFunctionValue(fieldToFetch, null, args, token);

                    var immediateResult = AddProjectionToResult(item, OneScore, FieldsToFetch, result, key, fieldVal);
                    if (immediateResult.Document != null)
                    {
                        return(immediateResult.Document);
                    }
                }
                else
                {
                    var val = match.GetResult(fieldToFetch.QueryField.ExpressionField.Compound[0].Value);

                    switch (val)
                    {
                    case Document d:
                    {
                        if (TryGetValue(fieldToFetch, d, null, null, null, null, out key, out fieldVal, token) == false)
                        {
                            continue;
                        }
                        d.EnsureMetadata();
                        var immediateResult = AddProjectionToResult(d, OneScore, FieldsToFetch, result, key, fieldVal);
                        if (immediateResult.Document != null)
                        {
                            return(immediateResult.Document);
                        }
                        break;
                    }

                    case BlittableJsonReaderObject bjro:
                    {
                        var doc = new Document {
                            Data = bjro
                        };
                        if (TryGetValue(fieldToFetch, doc, null, null, null, null, out key, out fieldVal, token) == false)
                        {
                            continue;
                        }
                        doc.EnsureMetadata();
                        var immediateResult = AddProjectionToResult(doc, OneScore, FieldsToFetch, result, key, fieldVal);
                        if (immediateResult.Document != null)
                        {
                            return(immediateResult.Document);
                        }
                        break;
                    }

                    case MatchCollection matches:
                        var array = new DynamicJsonArray();
                        foreach (var m in matches)
                        {
                            var djv = new DynamicJsonValue();
                            m.PopulateVertices(djv);

                            if (djv.Properties.Count == 0)
                            {
                                continue;
                            }

                            var matchJson = _context.ReadObject(djv, "graph/arg");

                            var doc = new Document {
                                Data = matchJson
                            };

                            if (TryGetValue(fieldToFetch, doc, null, null, null, null, out key, out fieldVal, token) == false)
                            {
                                continue;
                            }
                            doc.EnsureMetadata();
                            if (ReferenceEquals(doc, fieldVal))
                            {
                                fieldVal = doc.Data;
                            }

                            array.Add(fieldVal);
                        }
                        result[key] = array;
                        break;

                    case string s:
                        result[fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value] = s;
                        break;

                    default:
                        result[fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value] = null;
                        continue;
                    }
                }
            }

            return(new Document
            {
                Data = context.ReadObject(result, "projection result")
            });
        }
Exemplo n.º 22
0
        private DocumentItem ConvertRecordToDocumentItem(DocumentsOperationContext context, string[] csvReaderCurrentRecord, string[] csvReaderFieldHeaders, string collection)
        {
            try
            {
                var idStr = _hasId ? csvReaderCurrentRecord[_idIndex] : _hasCollection ? $"{csvReaderCurrentRecord[_collectionIndex]}/" : $"{collection}/";
                var data  = new DynamicJsonValue();
                for (int i = 0; i < csvReaderFieldHeaders.Length; i++)
                {
                    //ignoring reserved properties
                    if (csvReaderFieldHeaders[i][0] == '@')
                    {
                        if (_hasCollection && i == _collectionIndex)
                        {
                            SetCollectionForDocument(csvReaderCurrentRecord[_collectionIndex], data);
                        }
                        continue;
                    }

                    if (_nestedPropertyDictionary != null && _nestedPropertyDictionary.TryGetValue(i, out var segments))
                    {
                        var nestedData = data;
                        for (var j = 0; ; j++)
                        {
                            //last segment holds the data
                            if (j == segments.Length - 1)
                            {
                                nestedData[segments[j]] = ParseValue(csvReaderCurrentRecord[i]);
                                break; //we are done
                            }
                            //Creating the objects along the path if needed e.g. Foo.Bar.Name will create the 'Bar' oject if needed
                            if (nestedData[segments[j]] == null)
                            {
                                var tmpRef = new DynamicJsonValue();
                                nestedData[segments[j]] = tmpRef;
                                nestedData = tmpRef;
                            }
                            //We need to advance into the nested object, since it is not the last segment it must be of type 'DynamicJsonValue'
                            else
                            {
                                nestedData = (DynamicJsonValue)nestedData[segments[j]];
                            }
                        }
                        continue;
                    }
                    data[csvReaderFieldHeaders[i]] = ParseValue(csvReaderCurrentRecord[i]);
                }

                if (_hasCollection == false)
                {
                    SetCollectionForDocument(collection, data);
                }

                return(new DocumentItem
                {
                    Document = new Document
                    {
                        Data = context.ReadObject(data, idStr),
                        Id = context.GetLazyString(idStr),
                        ChangeVector = string.Empty,
                        Flags = DocumentFlags.None,
                        NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler,
                        LastModified = _database.Time.GetUtcNow(),
                    },
                    Attachments = null
                });
            }
            finally
            {
                foreach (var disposeMe in _disposibales)
                {
                    disposeMe.Dispose();
                }
                _disposibales.Clear();
            }
        }
Exemplo n.º 23
0
        public unsafe PatchResultData Apply(DocumentsOperationContext context,
                                            string documentKey,
                                            long?etag,
                                            PatchRequest patch,
                                            PatchRequest patchIfMissing,
                                            bool isTestOnly = false,
                                            bool skipPatchIfEtagMismatch = false)
        {
            var document = _database.DocumentsStorage.Get(context, documentKey);

            if (_logger.IsInfoEnabled)
            {
                _logger.Info(string.Format("Preparing to apply patch on ({0}). Document found?: {1}.", documentKey, document != null));
            }

            if (etag.HasValue && document != null && document.Etag != etag.Value)
            {
                System.Diagnostics.Debug.Assert(document.Etag > 0);

                if (skipPatchIfEtagMismatch)
                {
                    return(new PatchResultData
                    {
                        PatchResult = PatchResult.Skipped
                    });
                }

                if (_logger.IsInfoEnabled)
                {
                    _logger.Info($"Got concurrent exception while tried to patch the following document: {documentKey}");
                }
                throw new ConcurrencyException($"Could not patch document '{documentKey}' because non current etag was used")
                      {
                          ActualETag   = document.Etag,
                          ExpectedETag = etag.Value,
                      };
            }

            var patchRequest = patch;

            if (document == null)
            {
                if (patchIfMissing == null)
                {
                    if (_logger.IsInfoEnabled)
                    {
                        _logger.Info("Tried to patch a not exists document and patchIfMissing is null");
                    }

                    return(new PatchResultData
                    {
                        PatchResult = PatchResult.DocumentDoesNotExists
                    });
                }
                patchRequest = patchIfMissing;
            }
            var scope            = ApplySingleScript(context, document, isTestOnly, patchRequest);
            var modifiedDocument = context.ReadObject(scope.ToBlittable(scope.PatchObject.AsObject()),
                                                      documentKey, BlittableJsonDocumentBuilder.UsageMode.ToDisk);

            var result = new PatchResultData
            {
                PatchResult      = PatchResult.NotModified,
                OriginalDocument = document?.Data,
                DebugInfo        = scope.DebugInfo,
            };

            if (modifiedDocument == null)
            {
                if (_logger.IsInfoEnabled)
                {
                    _logger.Info($"After applying patch, modifiedDocument is null and document is null? {document == null}");
                }

                result.PatchResult = PatchResult.Skipped;
                return(result);
            }

            if (isTestOnly)
            {
                return(new PatchResultData
                {
                    PatchResult = PatchResult.Tested,
                    OriginalDocument = document?.Data,
                    ModifiedDocument = modifiedDocument,
                    DebugActions = scope.DebugActions.GetDebugActions(),
                    DebugInfo = scope.DebugInfo,
                });
            }

            var putResult = new DocumentsStorage.PutOperationResults();

            if (document == null)
            {
                putResult = _database.DocumentsStorage.Put(context, documentKey, null, modifiedDocument);
            }
            else
            {
                var isModified = document.Data.Size != modifiedDocument.Size;
                if (isModified == false) // optimization, if size different, no need to compute hash to check
                {
                    var originHash   = Hashing.XXHash64.Calculate(document.Data.BasePointer, (ulong)document.Data.Size);
                    var modifiedHash = Hashing.XXHash64.Calculate(modifiedDocument.BasePointer, (ulong)modifiedDocument.Size);
                    isModified = originHash != modifiedHash;
                }

                if (isModified)
                {
                    putResult = _database.DocumentsStorage.Put(context, document.Key, document.Etag,
                                                               modifiedDocument);
                    result.PatchResult = PatchResult.Patched;
                }
            }

            if (putResult.Etag != 0)
            {
                result.Etag       = putResult.Etag;
                result.Collection = putResult.Collection;
            }

            return(result);
        }
            public void Execute(DocumentsOperationContext context, Transaction indexWriteTransaction)
            {
                // delete

                foreach (string reduceOutputId in _deletedReduceOutputs)
                {
                    var referenceId = _index.OutputReduceToCollection.GetPatternGeneratedIdForReduceOutput(indexWriteTransaction, reduceOutputId);

                    if (referenceId == null)
                    {
                        continue;
                    }

                    if (_idsToDeleteByReferenceDocumentId.TryGetValue(referenceId, out var values) == false)
                    {
                        values = new HashSet <string>(1, StringComparer.OrdinalIgnoreCase);

                        _idsToDeleteByReferenceDocumentId.Add(referenceId, values);
                    }

                    values.Add(reduceOutputId);
                }

                foreach (var reduceReferenceIdToReduceOutputIds in _idsToDeleteByReferenceDocumentId)
                {
                    using (var referenceDocument = _database.DocumentsStorage.Get(context, reduceReferenceIdToReduceOutputIds.Key))
                    {
                        if (referenceDocument == null)
                        {
                            continue;
                        }

                        if (referenceDocument.Data.TryGet(nameof(OutputReduceToCollectionReference.ReduceOutputs), out BlittableJsonReaderArray ids) == false)
                        {
                            ThrowReduceOutputsPropertyNotFound(referenceDocument.Id);
                        }

                        var idsToRemove = reduceReferenceIdToReduceOutputIds.Value;

                        if (idsToRemove.Count >= ids.Length)
                        {
                            Debug.Assert(ids.All(x => idsToRemove.Contains(x.ToString())), $"Found ID in {nameof(idsToRemove)} that aren't in {nameof(ids)}");

                            foreach (object deletedId in ids)
                            {
                                _index.OutputReduceToCollection.DeletePatternGeneratedIdForReduceOutput(indexWriteTransaction, deletedId.ToString());
                            }

                            _database.DocumentsStorage.Delete(context, referenceDocument.Id, null);
                            continue;
                        }

                        if (ids.Modifications == null)
                        {
                            ids.Modifications = new DynamicJsonArray();
                        }

                        var indexesToRemove = new List <int>();

                        for (int i = ids.Length - 1; i >= 0; i--)
                        {
                            var id = ids[i].ToString();

                            if (idsToRemove.Contains(id))
                            {
                                indexesToRemove.Add(i);
                            }

                            if (idsToRemove.Count == indexesToRemove.Count)
                            {
                                break;
                            }
                        }

                        foreach (int toRemove in indexesToRemove)
                        {
                            ids.Modifications.RemoveAt(toRemove);
                        }

                        using (var doc = context.ReadObject(referenceDocument.Data, referenceDocument.Id))
                        {
                            _database.DocumentsStorage.Put(context, referenceDocument.Id, null, doc);

                            foreach (var idToRemove in idsToRemove)
                            {
                                _index.OutputReduceToCollection.DeletePatternGeneratedIdForReduceOutput(indexWriteTransaction, idToRemove);
                            }
                        }
                    }
                }

                // put

                foreach (var referencesOfReduceOutput in _referencesOfReduceOutputs)
                {
                    using (var existingReferenceDocument = _database.DocumentsStorage.Get(context, referencesOfReduceOutput.Key))
                    {
                        var uniqueIds = referencesOfReduceOutput.Value;

                        if (existingReferenceDocument != null)
                        {
                            if (existingReferenceDocument.Data.TryGet(nameof(OutputReduceToCollectionReference.ReduceOutputs), out BlittableJsonReaderArray existingIds) == false)
                            {
                                ThrowReduceOutputsPropertyNotFound(existingReferenceDocument.Id);
                            }

                            foreach (object id in existingIds)
                            {
                                uniqueIds.Add(id.ToString());
                            }
                        }

                        var referenceDoc = new DynamicJsonValue
                        {
                            [nameof(OutputReduceToCollectionReference.ReduceOutputs)] = new DynamicJsonArray(uniqueIds),
                            [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                            {
                                [Constants.Documents.Metadata.Collection] = _referencesCollectionName ?? $"{_outputReduceToCollection}/References"
                            }
                        };

                        using (var referenceJson = context.ReadObject(referenceDoc, "reference-of-reduce-output", BlittableJsonDocumentBuilder.UsageMode.ToDisk))
                        {
                            _database.DocumentsStorage.Delete(context, referencesOfReduceOutput.Key, null);

                            _database.DocumentsStorage.Put(context, referencesOfReduceOutput.Key, null, referenceJson,
                                                           flags: DocumentFlags.Artificial | DocumentFlags.FromIndex);

                            foreach (var reduceOutputId in referencesOfReduceOutput.Value)
                            {
                                _index.OutputReduceToCollection.AddPatternGeneratedIdForReduceOutput(indexWriteTransaction, reduceOutputId,
                                                                                                     referencesOfReduceOutput.Key);
                            }
                        }
                    }
                }
            }
Exemplo n.º 25
0
        private void CleanupReferenceDocuments(DocumentsOperationContext context, Dictionary <string, HashSet <string> > idsToDeleteByReferenceDocumentId)
        {
            foreach (var reduceReferenceIdToReduceOutputIds in idsToDeleteByReferenceDocumentId)
            {
                using (var referenceDocument = _database.DocumentsStorage.Get(context, reduceReferenceIdToReduceOutputIds.Key))
                {
                    if (referenceDocument == null)
                    {
                        continue;
                    }

                    if (referenceDocument.Data.TryGet(nameof(OutputReduceToCollectionReference.ReduceOutputs), out BlittableJsonReaderArray ids) == false)
                    {
                        ThrowIdsPropertyNotFound(referenceDocument.Id);
                    }

                    var idsToRemove = reduceReferenceIdToReduceOutputIds.Value;

                    if (ids.Modifications == null)
                    {
                        ids.Modifications = new DynamicJsonArray();
                    }

                    var indexesToRemove = new List <int>();

                    for (int i = ids.Length - 1; i >= 0; i--)
                    {
                        var id = ids[i].ToString();

                        if (idsToRemove.Contains(id))
                        {
                            indexesToRemove.Add(i);
                        }

                        if (idsToRemove.Count == indexesToRemove.Count)
                        {
                            break;
                        }
                    }

                    foreach (int toRemove in indexesToRemove)
                    {
                        ids.Modifications.RemoveAt(toRemove);
                    }

                    using (var doc = context.ReadObject(referenceDocument.Data, referenceDocument.Id))
                    {
                        if (doc.TryGet(nameof(OutputReduceToCollectionReference.ReduceOutputs), out BlittableJsonReaderArray updatedIds) == false)
                        {
                            ThrowIdsPropertyNotFound(referenceDocument.Id);
                        }

                        if (updatedIds.Length == 0)
                        {
                            _database.DocumentsStorage.Delete(context, referenceDocument.Id, null);
                        }
                        else
                        {
                            _database.DocumentsStorage.Put(context, referenceDocument.Id, null, doc);
                        }
                    }
                }
            }
        }
Exemplo n.º 26
0
        private async Task <(List <Match> Matches, GraphQueryPlan QueryPlan, bool NotModified)> GetQueryResults(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token, bool collectIntermediateResults = false)
        {
            var q  = query.Metadata.Query;
            var qp = new GraphQueryPlan(query, documentsContext, existingResultEtag, token, Database)
            {
                CollectIntermediateResults = collectIntermediateResults
            };

            qp.BuildQueryPlan();
            qp.OptimizeQueryPlan(); //TODO: audit optimization

            if (query.WaitForNonStaleResults)
            {
                qp.IsStale = await qp.WaitForNonStaleResults();
            }
            else
            {
                await qp.CreateAutoIndexesAndWaitIfNecessary();
            }

            //for the case where we don't wait for non stale results we will override IsStale in the QueryQueryStep steps

            if (documentsContext.Transaction == null || documentsContext.Transaction.Disposed)
            {
                documentsContext.OpenReadTransaction();
            }

            qp.ResultEtag = DocumentsStorage.ReadLastEtag(documentsContext.Transaction.InnerTransaction);
            if (existingResultEtag.HasValue)
            {
                if (qp.ResultEtag == existingResultEtag)
                {
                    return(null, null, true);
                }
            }
            await qp.Initialize();

            var matchResults = qp.Execute();

            if (query.Metadata.OrderBy != null)
            {
                Sort(matchResults, query.Metadata.OrderBy, Database.Name, query.Query);
            }

            var filter = q.GraphQuery.Where;

            if (filter != null)
            {
                for (int i = 0; i < matchResults.Count; i++)
                {
                    var resultAsJson = new DynamicJsonValue();
                    matchResults[i].PopulateVertices(resultAsJson);

                    using (var result = documentsContext.ReadObject(resultAsJson, "graph/result"))
                    {
                        if (filter.IsMatchedBy(result, query.QueryParameters) == false)
                        {
                            matchResults[i] = default;
                        }
                    }
                }
            }

            if (query.Start > 0)
            {
                matchResults.RemoveRange(0, Math.Min(query.Start, matchResults.Count));
            }

            if (query.PageSize < matchResults.Count)
            {
                matchResults.RemoveRange(query.PageSize, matchResults.Count - query.PageSize);
            }
            return(matchResults, qp, false);
        }
Exemplo n.º 27
0
        /// <summary>
        /// Iterates on a batch in document collection, process it and send documents if found any match
        /// </summary>
        /// <param name="docsContext"></param>
        /// <param name="sendingCurrentBatchStopwatch"></param>
        /// <returns>Whether succeeded finding any documents to send</returns>
        private async Task <bool> TrySendingBatchToClient(DocumentsOperationContext docsContext, Stopwatch sendingCurrentBatchStopwatch)
        {
            bool anyDocumentsSentInCurrentIteration = false;
            int  docsToFlush = 0;

            using (var writer = new BlittableJsonTextWriter(docsContext, _buffer))
            {
                using (docsContext.OpenReadTransaction())
                {
                    foreach (var result in _documentsFetcher.GetDataToSend(docsContext, _startEtag))
                    {
                        _startEtag        = result.Doc.Etag;
                        _lastChangeVector = string.IsNullOrEmpty(SubscriptionState.ChangeVectorForNextBatchStartingPoint)
                            ? result.Doc.ChangeVector
                            : ChangeVectorUtils.MergeVectors(result.Doc.ChangeVector, SubscriptionState.ChangeVectorForNextBatchStartingPoint);

                        if (result.Doc.Data == null)
                        {
                            if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
                            {
                                await SendHeartBeat();

                                sendingCurrentBatchStopwatch.Restart();
                            }

                            continue;
                        }

                        anyDocumentsSentInCurrentIteration = true;
                        writer.WriteStartObject();

                        writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(TypeSegment));
                        writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(DataSegment));
                        writer.WriteComma();
                        writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(DataSegment));
                        result.Doc.EnsureMetadata();

                        if (result.Exception != null)
                        {
                            var metadata = result.Doc.Data[Client.Constants.Documents.Metadata.Key];
                            writer.WriteValue(BlittableJsonToken.StartObject,
                                              docsContext.ReadObject(new DynamicJsonValue
                            {
                                [Client.Constants.Documents.Metadata.Key] = metadata
                            }, result.Doc.Id)
                                              );
                            writer.WriteComma();
                            writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(ExceptionSegment));
                            writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(result.Exception.ToString()));
                        }
                        else
                        {
                            writer.WriteDocument(docsContext, result.Doc, metadataOnly: false);
                        }

                        writer.WriteEndObject();
                        docsToFlush++;

                        // perform flush for current batch after 1000ms of running or 1 MB
                        if (_buffer.Length > Constants.Size.Megabyte ||
                            sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
                        {
                            if (docsToFlush > 0)
                            {
                                await FlushDocsToClient(writer, docsToFlush);

                                docsToFlush = 0;
                                sendingCurrentBatchStopwatch.Restart();
                            }
                            else
                            {
                                await SendHeartBeat();
                            }
                        }
                    }
                }

                if (anyDocumentsSentInCurrentIteration)
                {
                    docsContext.Write(writer, new DynamicJsonValue
                    {
                        [nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.EndOfBatch)
                    });

                    await FlushDocsToClient(writer, docsToFlush, true);

                    if (_logger.IsInfoEnabled)
                    {
                        _logger.Info(
                            $"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}");
                    }
                }
            }
            return(anyDocumentsSentInCurrentIteration);
        }
Exemplo n.º 28
0
        public void Put(DocumentsOperationContext context, string id, BlittableJsonReaderObject document,
                        DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags, string changeVector, long lastModifiedTicks,
                        RevisionsCollectionConfiguration configuration = null, CollectionName collectionName = null)
        {
            Debug.Assert(changeVector != null, "Change vector must be set");
            Debug.Assert(lastModifiedTicks != DateTime.MinValue.Ticks, "last modified ticks must be set");

            BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);

            if (collectionName == null)
            {
                collectionName = _database.DocumentsStorage.ExtractCollectionName(context, document);
            }
            if (configuration == null)
            {
                configuration = GetRevisionsConfiguration(collectionName.Name);
            }

            using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
            {
                var fromReplication = (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) == NonPersistentDocumentFlags.FromReplication;

                var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName);

                // We want the revision's attachments to have a lower etag than the revision itself
                if ((flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
                {
                    if (flags.Contain(DocumentFlags.Revision) == false)
                    {
                        using (Slice.From(context.Allocator, changeVector, out Slice changeVectorSlice))
                        {
                            if (table.VerifyKeyExists(changeVectorSlice) == false)
                            {
                                _documentsStorage.AttachmentsStorage.RevisionAttachments(context, lowerId, changeVectorSlice);
                            }
                        }
                    }
                }

                if (document.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
                    metadata.TryGet(Constants.Documents.Metadata.Counters, out BlittableJsonReaderArray counterNames))
                {
                    var dvj = new DynamicJsonValue();
                    for (var i = 0; i < counterNames.Length; i++)
                    {
                        var counter = counterNames[i].ToString();
                        var val     = _documentsStorage.CountersStorage.GetCounterValue(context, id, counter);
                        if (val == null)
                        {
                            continue;
                        }
                        dvj[counter] = val.Value;
                    }

                    metadata.Modifications = new DynamicJsonValue(metadata)
                    {
                        [Constants.Documents.Metadata.RevisionCounters] = dvj
                    };
                    metadata.Modifications.Remove(Constants.Documents.Metadata.Counters);
                    document.Modifications = new DynamicJsonValue(document)
                    {
                        [Constants.Documents.Metadata.Key] = metadata
                    };

                    document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
                }

                if (fromReplication)
                {
                    void PutFromRevisionIfChangeVectorIsGreater()
                    {
                        bool             hasDoc;
                        TableValueReader tvr;

                        try
                        {
                            hasDoc = _documentsStorage.GetTableValueReaderForDocument(context, lowerId, throwOnConflict: true, tvr: out tvr);
                        }
                        catch (DocumentConflictException)
                        {
                            // Do not modify the document.
                            return;
                        }

                        if (hasDoc == false)
                        {
                            PutFromRevision();
                            return;
                        }

                        var docChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref tvr);

                        if (ChangeVectorUtils.GetConflictStatus(changeVector, docChangeVector) == ConflictStatus.Update)
                        {
                            PutFromRevision();
                        }

                        void PutFromRevision()
                        {
                            _documentsStorage.Put(context, id, null, document, lastModifiedTicks, changeVector,
                                                  flags & ~DocumentFlags.Revision, nonPersistentFlags | NonPersistentDocumentFlags.FromRevision);
                        }
                    }

                    PutFromRevisionIfChangeVectorIsGreater();
                }

                flags |= DocumentFlags.Revision;
                var newEtag          = _database.DocumentsStorage.GenerateNextEtag();
                var newEtagSwapBytes = Bits.SwapBytes(newEtag);

                using (table.Allocate(out TableValueBuilder tvb))
                    using (Slice.From(context.Allocator, changeVector, out var cv))
                    {
                        tvb.Add(cv.Content.Ptr, cv.Size);
                        tvb.Add(lowerId);
                        tvb.Add(SpecialChars.RecordSeparator);
                        tvb.Add(newEtagSwapBytes);
                        tvb.Add(idPtr);
                        tvb.Add(document.BasePointer, document.Size);
                        tvb.Add((int)flags);
                        tvb.Add(NotDeletedRevisionMarker);
                        tvb.Add(lastModifiedTicks);
                        tvb.Add(context.GetTransactionMarker());
                        if (flags.Contain(DocumentFlags.Resolved))
                        {
                            tvb.Add((int)DocumentFlags.Resolved);
                        }
                        else
                        {
                            tvb.Add(0);
                        }
                        tvb.Add(Bits.SwapBytes(lastModifiedTicks));
                        var isNew = table.Set(tvb);
                        if (isNew == false)
                        {
                            // It might be just an update from replication as we call this twice, both for the doc delete and for deleteRevision.
                            return;
                        }
                    }

                DeleteOldRevisions(context, table, lowerId, collectionName, configuration, nonPersistentFlags, changeVector, lastModifiedTicks);
            }
        }
Exemplo n.º 29
0
            public override void Execute(DocumentsOperationContext context, RavenTransaction tx)
            {
                var hiLoDocumentKey = RavenKeyGeneratorsHilo + Key;
                var prefix          = Key + Separator;

                long oldMax = 0;
                var  newDoc = new DynamicJsonValue();
                BlittableJsonReaderObject hiloDocReader = null, serverPrefixDocReader = null;

                try
                {
                    try
                    {
                        serverPrefixDocReader = Database.DocumentsStorage.Get(context, RavenKeyServerPrefix)?.Data;
                        hiloDocReader         = Database.DocumentsStorage.Get(context, hiLoDocumentKey)?.Data;
                    }
                    catch (DocumentConflictException e)
                    {
                        throw new InvalidDataException(@"Failed to fetch HiLo document due to a conflict 
                                                            on the document. This shouldn't happen, since
                                                            it this conflict should've been resolved during replication.
                                                             This exception should not happen and is likely a bug.", e);
                    }

                    string serverPrefix;
                    if (serverPrefixDocReader != null &&
                        serverPrefixDocReader.TryGet("ServerPrefix", out serverPrefix))
                    {
                        prefix += serverPrefix;
                    }

                    if (hiloDocReader != null)
                    {
                        hiloDocReader.TryGet("Max", out oldMax);
                        var prop = new BlittableJsonReaderObject.PropertyDetails();
                        for (var i = 0; i < hiloDocReader.Count; i++)
                        {
                            hiloDocReader.GetPropertyByIndex(0, ref prop);
                            if (prop.Name == "Max")
                            {
                                continue;
                            }
                            newDoc[prop.Name] = prop.Value;
                        }
                    }
                }

                finally
                {
                    serverPrefixDocReader?.Dispose();
                    hiloDocReader?.Dispose();
                }
                oldMax = Math.Max(oldMax, LastRangeMax);

                newDoc["Max"] = oldMax + Capacity;

                using (
                    var freshHilo = context.ReadObject(newDoc, hiLoDocumentKey,
                                                       BlittableJsonDocumentBuilder.UsageMode.ToDisk))
                {
                    Database.DocumentsStorage.Put(context, hiLoDocumentKey, null, freshHilo);
                }

                OldMax = oldMax;
                Prefix = prefix;
            }