public static void WriteFacetQuery(this BlittableJsonTextWriter writer, DocumentConventions conventions, JsonOperationContext context, FacetQuery query)
 {
     writer.WriteObject(EntityToBlittable.ConvertEntityToBlittable(query, conventions, context));
 }
Esempio n. 2
0
        public async Task SubscriptionShouldRespectDocumentsWithCompressedData()
        {
            using (var documentStore = this.GetDocumentStore())
            {
                Server.ServerStore.Observer.Suspended = true;
                var originalDoc = new Doc
                {
                    Id            = "doc/1",
                    StrVal        = new string(Enumerable.Repeat('.', 129).ToArray()),
                    LongByteArray = Enumerable.Repeat((byte)2, 1024).ToArray()
                };

                using (var session = documentStore.OpenAsyncSession())
                {
                    await session.StoreAsync(originalDoc);

                    await session.SaveChangesAsync();
                }

                var database = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(documentStore.Database);

                using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                    using (context.OpenReadTransaction())
                    {
                        var          doc = database.DocumentsStorage.Get(context, "doc/1");
                        MemoryStream ms  = new MemoryStream();
                        using (var newContext = JsonOperationContext.ShortTermSingleUse())
                            using (var writer = new BlittableJsonTextWriter(newContext, ms))
                            {
                                writer.WriteDocument(newContext, doc, metadataOnly: false);
                                writer.Flush();
                                var bjro             = GetReaderFromMemoryStream(ms, context);
                                var desereializedDoc = (Doc)EntityToBlittable.ConvertToEntity(typeof(Doc), null, bjro, DocumentConventions.Default);

                                Assert.Equal(originalDoc.StrVal, desereializedDoc.StrVal);
                                Assert.Equal(originalDoc.LongByteArray, originalDoc.LongByteArray);
                            }
                    }

                var subscriptionCreationParams = new SubscriptionCreationOptions
                {
                    Query = "from Docs",
                };

                var subsId = await documentStore.Subscriptions.CreateAsync(subscriptionCreationParams).ConfigureAwait(false);

                var amre = new AsyncManualResetEvent();
                using (var subscription = documentStore.Subscriptions.GetSubscriptionWorker <Doc>(new SubscriptionWorkerOptions(subsId)
                {
                    TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
                }))
                {
                    var t = subscription.Run(batch =>
                    {
                        var receivedDoc = batch.Items.First().Result;
                        Assert.Equal(originalDoc.LongByteArray, receivedDoc.LongByteArray);
                        Assert.Equal(originalDoc.StrVal, receivedDoc.StrVal);
                        amre.Set();
                    });

                    try
                    {
                        Assert.True(await amre.WaitAsync(TimeSpan.FromSeconds(60)));
                    }
                    catch
                    {
                        if (t.IsFaulted)
                        {
                            t.Wait();
                        }
                        throw;
                    }
                }
            }
        }
        private void WriteDocument(BlittableJsonTextWriter writer, JsonOperationContext context, Document document, HashSet <string> propertiesPreviewToSend, HashSet <string> fullPropertiesToSend)
        {
            writer.WriteStartObject();

            document.Data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata);

            bool first = true;

            var arrayStubsJson  = new DynamicJsonValue();
            var objectStubsJson = new DynamicJsonValue();
            var trimmedValue    = new HashSet <LazyStringValue>();

            var prop = new BlittableJsonReaderObject.PropertyDetails();

            using (var buffers = document.Data.GetPropertiesByInsertionOrder())
            {
                for (int i = 0; i < buffers.Properties.Count; i++)
                {
                    document.Data.GetPropertyByIndex(buffers.Properties.Array[i + buffers.Properties.Offset], ref prop);
                    var sendFull = fullPropertiesToSend.Contains(prop.Name);
                    if (sendFull || propertiesPreviewToSend.Contains(prop.Name))
                    {
                        var strategy = sendFull ? ValueWriteStrategy.Passthrough : FindWriteStrategy(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);

                        if (strategy == ValueWriteStrategy.Passthrough || strategy == ValueWriteStrategy.Trim)
                        {
                            if (first == false)
                            {
                                writer.WriteComma();
                            }
                            first = false;
                        }

                        switch (strategy)
                        {
                        case ValueWriteStrategy.Passthrough:
                            writer.WritePropertyName(prop.Name);
                            writer.WriteValue(prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
                            break;

                        case ValueWriteStrategy.SubstituteWithArrayStub:
                            arrayStubsJson[prop.Name] = ((BlittableJsonReaderArray)prop.Value).Length;
                            break;

                        case ValueWriteStrategy.SubstituteWithObjectStub:
                            objectStubsJson[prop.Name] = ((BlittableJsonReaderObject)prop.Value).Count;
                            break;

                        case ValueWriteStrategy.Trim:
                            writer.WritePropertyName(prop.Name);
                            WriteTrimmedValue(writer, prop.Token & BlittableJsonReaderBase.TypesMask, prop.Value);
                            trimmedValue.Add(prop.Name);
                            break;
                        }
                    }
                }
            }
            if (first == false)
            {
                writer.WriteComma();
            }

            var extraMetadataProperties = new DynamicJsonValue
            {
                [ObjectStubsKey]  = objectStubsJson,
                [ArrayStubsKey]   = arrayStubsJson,
                [TrimmedValueKey] = new DynamicJsonArray(trimmedValue)
            };

            if (metadata != null)
            {
                metadata.Modifications = extraMetadataProperties;
                metadata = context.ReadObject(metadata, document.Id);
            }
            else
            {
                metadata = context.ReadObject(extraMetadataProperties, document.Id);
            }

            writer.WriteMetadata(document, metadata);
            writer.WriteEndObject();
        }
Esempio n. 4
0
        public async Task ApplyCommand()
        {
            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
            {
                if (ServerStore.IsLeader() == false)
                {
                    throw new NoLeaderException("Not a leader, cannot accept commands.");
                }

                HttpContext.Response.Headers["Reached-Leader"] = "true";

                var commandJson = await context.ReadForMemoryAsync(RequestBodyStream(), "external/rachis/command");

                CommandBase command;
                try
                {
                    command = CommandBase.CreateFrom(commandJson);
                }
                catch (InvalidOperationException e)
                {
                    RequestRouter.AssertClientVersion(HttpContext, e);
                    throw;
                }

                switch (command)
                {
                case AddOrUpdateCompareExchangeBatchCommand batchCmpExchange:
                    batchCmpExchange.ContextToWriteResult = context;
                    break;

                case CompareExchangeCommandBase cmpExchange:
                    cmpExchange.ContextToWriteResult = context;
                    break;
                }

                var isClusterAdmin = IsClusterAdmin();
                command.VerifyCanExecuteCommand(ServerStore, context, isClusterAdmin);

                var(etag, result) = await ServerStore.Engine.PutAsync(command);

                HttpContext.Response.StatusCode = (int)HttpStatusCode.OK;
                var ms = context.CheckoutMemoryStream();
                try
                {
                    using (var writer = new BlittableJsonTextWriter(context, ms))
                    {
                        context.Write(writer, new DynamicJsonValue
                        {
                            [nameof(ServerStore.PutRaftCommandResult.RaftCommandIndex)] = etag,
                            [nameof(ServerStore.PutRaftCommandResult.Data)]             = result
                        });
                        writer.Flush();
                    }
                    // now that we know that we properly serialized it
                    ms.Position = 0;
                    await ms.CopyToAsync(ResponseBodyStream());
                }
                finally
                {
                    context.ReturnMemoryStream(ms);
                }
            }
        }
Esempio n. 5
0
        public async Task Try()
        {
            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
            {
                var json = await context.ReadForMemoryAsync(RequestBodyStream(), null);

                var tryout = JsonDeserializationServer.SubscriptionTryout(json);

                var(collection, (script, functions), revisions) = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query);
                SubscriptionPatchDocument patch = null;
                if (string.IsNullOrEmpty(script) == false)
                {
                    patch = new SubscriptionPatchDocument(script, functions);
                }

                if (collection == null)
                {
                    throw new ArgumentException("Collection must be specified");
                }

                var pageSize = GetIntValueQueryString("pageSize") ?? 1;

                var state = new SubscriptionState
                {
                    ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector,
                    Query = tryout.Query
                };

                var fetcher = new SubscriptionDocumentsFetcher(Database, pageSize, -0x42,
                                                               new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), collection, revisions, state, patch);

                if (Enum.TryParse(
                        tryout.ChangeVector,
                        out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
                {
                    switch (changeVectorSpecialValue)
                    {
                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange:
                        state.ChangeVectorForNextBatchStartingPoint = null;
                        break;

                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
                        state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context, collection);
                        break;
                    }
                }

                using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    writer.WriteStartObject();
                    writer.WritePropertyName("Results");
                    writer.WriteStartArray();

                    using (context.OpenReadTransaction())
                    {
                        var first = true;

                        foreach (var itemDetails in fetcher.GetDataToSend(context, 0))
                        {
                            if (itemDetails.Doc.Data == null)
                            {
                                continue;
                            }

                            if (first == false)
                            {
                                writer.WriteComma();
                            }

                            if (itemDetails.Exception == null)
                            {
                                writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false);
                            }
                            else
                            {
                                var docWithExcepton = new DocumentWithException
                                {
                                    Exception    = itemDetails.Exception.ToString(),
                                    ChangeVector = itemDetails.Doc.ChangeVector,
                                    Id           = itemDetails.Doc.Id,
                                    DocumentData = itemDetails.Doc.Data
                                };
                                writer.WriteObject(context.ReadObject(docWithExcepton.ToJson(), ""));
                            }

                            first = false;
                        }
                    }

                    writer.WriteEndArray();
                    writer.WriteEndObject();
                }
            }
        }
Esempio n. 6
0
        public Task Stats()
        {
            var documentId = GetStringQueryString("docId");

            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                using (context.OpenReadTransaction())
                {
                    var document = Database.DocumentsStorage.Get(context, documentId, DocumentFields.Data);
                    if (document == null)
                    {
                        HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
                        return(Task.CompletedTask);
                    }

                    var timeSeriesNames = new List <string>();

                    if (document.TryGetMetadata(out var metadata))
                    {
                        if (metadata.TryGet(Constants.Documents.Metadata.TimeSeries, out BlittableJsonReaderArray timeSeries) && timeSeries != null)
                        {
                            foreach (object name in timeSeries)
                            {
                                if (name == null)
                                {
                                    continue;
                                }

                                if (name is string || name is LazyStringValue || name is LazyCompressedStringValue)
                                {
                                    timeSeriesNames.Add(name.ToString());
                                }
                            }
                        }
                    }

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        writer.WriteStartObject();

                        writer.WritePropertyName(nameof(TimeSeriesStatistics.DocumentId));
                        writer.WriteString(documentId);
                        writer.WriteComma();

                        writer.WritePropertyName(nameof(TimeSeriesStatistics.TimeSeries));

                        writer.WriteStartArray();

                        var first = true;
                        foreach (var tsName in timeSeriesNames)
                        {
                            if (first == false)
                            {
                                writer.WriteComma();
                            }
                            first = false;

                            var stats = Database.DocumentsStorage.TimeSeriesStorage.Stats.GetStats(context, documentId, tsName);

                            writer.WriteStartObject();

                            writer.WritePropertyName(nameof(TimeSeriesItemDetail.Name));
                            writer.WriteString(tsName);

                            writer.WriteComma();

                            writer.WritePropertyName(nameof(TimeSeriesItemDetail.NumberOfEntries));
                            writer.WriteInteger(stats.Count);

                            writer.WriteComma();

                            writer.WritePropertyName(nameof(TimeSeriesItemDetail.StartDate));
                            writer.WriteDateTime(stats.Start, isUtc: true);

                            writer.WriteComma();

                            writer.WritePropertyName(nameof(TimeSeriesItemDetail.EndDate));
                            writer.WriteDateTime(stats.End, isUtc: true);

                            writer.WriteEndObject();
                        }

                        writer.WriteEndArray();

                        writer.WriteEndObject();
                    }
                }

            return(Task.CompletedTask);
        }
        public Task GetCertificates()
        {
            var thumbprint    = GetStringQueryString("thumbprint", required: false);
            var showSecondary = GetBoolValueQueryString("secondary", required: false) ?? false;

            var start    = GetStart();
            var pageSize = GetPageSize();

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
            {
                var certificates = new List <(string ItemName, BlittableJsonReaderObject Value)>();
                try
                {
                    if (string.IsNullOrEmpty(thumbprint))
                    {
                        if (ServerStore.CurrentRachisState == RachisState.Passive)
                        {
                            List <string> localCertKeys;
                            using (context.OpenReadTransaction())
                                localCertKeys = ServerStore.Cluster.GetCertificateKeysFromLocalState(context).ToList();

                            var serverCertKey = Constants.Certificates.Prefix + Server.Certificate.Certificate?.Thumbprint;
                            if (Server.Certificate.Certificate != null && localCertKeys.Contains(serverCertKey) == false)
                            {
                                // Since we didn't go through EnsureNotPassive(), the server certificate is not registered yet so we'll add it to the local state.
                                var serverCertDef = new CertificateDefinition
                                {
                                    Name              = "Server Certificate",
                                    Certificate       = Convert.ToBase64String(Server.Certificate.Certificate.Export(X509ContentType.Cert)),
                                    Permissions       = new Dictionary <string, DatabaseAccess>(),
                                    SecurityClearance = SecurityClearance.ClusterNode,
                                    Thumbprint        = Server.Certificate.Certificate.Thumbprint,
                                    NotAfter          = Server.Certificate.Certificate.NotAfter
                                };

                                var serverCert = context.ReadObject(serverCertDef.ToJson(), "Server/Certificate/Definition");
                                using (var tx = context.OpenWriteTransaction())
                                {
                                    ServerStore.Cluster.PutLocalState(context, Constants.Certificates.Prefix + Server.Certificate.Certificate.Thumbprint, serverCert);
                                    tx.Commit();
                                }
                                certificates.Add((serverCertKey, serverCert));
                            }

                            foreach (var localCertKey in localCertKeys)
                            {
                                BlittableJsonReaderObject localCertificate;
                                using (context.OpenReadTransaction())
                                    localCertificate = ServerStore.Cluster.GetLocalState(context, localCertKey);

                                if (localCertificate == null)
                                {
                                    continue;
                                }

                                var def = JsonDeserializationServer.CertificateDefinition(localCertificate);

                                if (showSecondary || string.IsNullOrEmpty(def.CollectionPrimaryKey))
                                {
                                    certificates.Add((localCertKey, localCertificate));
                                }
                                else
                                {
                                    localCertificate.Dispose();
                                }
                            }
                        }
                        else
                        {
                            using (context.OpenReadTransaction())
                            {
                                foreach (var item in ServerStore.Cluster.ItemsStartingWith(context, Constants.Certificates.Prefix, start, pageSize))
                                {
                                    var def = JsonDeserializationServer.CertificateDefinition(item.Value);

                                    if (showSecondary || string.IsNullOrEmpty(def.CollectionPrimaryKey))
                                    {
                                        certificates.Add(item);
                                    }
                                    else
                                    {
                                        item.Value.Dispose();
                                    }
                                }
                            }
                        }
                    }
                    else
                    {
                        using (context.OpenReadTransaction())
                        {
                            var key = Constants.Certificates.Prefix + thumbprint;

                            var certificate = ServerStore.CurrentRachisState == RachisState.Passive
                                ? ServerStore.Cluster.GetLocalState(context, key)
                                : ServerStore.Cluster.Read(context, key);

                            if (certificate == null)
                            {
                                HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
                                return(Task.CompletedTask);
                            }

                            var definition = JsonDeserializationServer.CertificateDefinition(certificate);
                            if (string.IsNullOrEmpty(definition.CollectionPrimaryKey) == false)
                            {
                                certificate = ServerStore.Cluster.Read(context, definition.CollectionPrimaryKey);
                                if (certificate == null)
                                {
                                    HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
                                    return(Task.CompletedTask);
                                }
                            }

                            certificates.Add((key, certificate));
                        }
                    }

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        writer.WriteStartObject();
                        writer.WriteArray(context, "Results", certificates.ToArray(), (w, c, cert) =>
                        {
                            c.Write(w, cert.Value);
                        });
                        writer.WriteComma();
                        writer.WritePropertyName("LoadedServerCert");
                        writer.WriteString(Server.Certificate.Certificate?.Thumbprint);
                        writer.WriteEndObject();
                    }
                }
                finally
                {
                    foreach (var cert in certificates)
                    {
                        cert.Value?.Dispose();
                    }
                }
            }

            return(Task.CompletedTask);
        }
Esempio n. 8
0
        public static void WriteIndexQuery(this BlittableJsonTextWriter writer, JsonOperationContext context, IndexQueryServerSide query)
        {
            writer.WriteStartObject();

            writer.WritePropertyName(nameof(query.CutoffEtag));
            if (query.CutoffEtag.HasValue)
            {
                writer.WriteInteger(query.CutoffEtag.Value);
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName(nameof(query.ExplainScores));
            writer.WriteBool(query.ExplainScores);
            writer.WriteComma();

            writer.WritePropertyName(nameof(query.PageSize));
            writer.WriteInteger(query.PageSize);
            writer.WriteComma();

            writer.WritePropertyName(nameof(query.Query));
            if (query.Query != null)
            {
                writer.WriteString(query.Query);
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName(nameof(query.ShowTimings));
            writer.WriteBool(query.ShowTimings);
            writer.WriteComma();

            writer.WritePropertyName(nameof(query.SkipDuplicateChecking));
            writer.WriteBool(query.SkipDuplicateChecking);
            writer.WriteComma();

            writer.WritePropertyName(nameof(query.Start));
            writer.WriteInteger(query.Start);
            writer.WriteComma();

            writer.WritePropertyName(nameof(query.WaitForNonStaleResults));
            writer.WriteBool(query.WaitForNonStaleResults);
            writer.WriteComma();

            writer.WritePropertyName(nameof(query.WaitForNonStaleResultsTimeout));
            if (query.WaitForNonStaleResultsTimeout.HasValue)
            {
                writer.WriteString(query.WaitForNonStaleResultsTimeout.Value.ToString());
            }
            else
            {
                writer.WriteNull();
            }

            writer.WriteEndObject();
        }
Esempio n. 9
0
        public static void WriteDatabaseStatistics(this BlittableJsonTextWriter writer, JsonOperationContext context, DatabaseStatistics statistics)
        {
            writer.WriteStartObject();

            writer.WritePropertyName(nameof(statistics.CountOfIndexes));
            writer.WriteInteger(statistics.CountOfIndexes);
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.CountOfDocuments));
            writer.WriteInteger(statistics.CountOfDocuments);
            writer.WriteComma();

            if (statistics.CountOfRevisionDocuments > 0)
            {
                writer.WritePropertyName(nameof(statistics.CountOfRevisionDocuments));
                writer.WriteInteger(statistics.CountOfRevisionDocuments);
                writer.WriteComma();
            }

            writer.WritePropertyName(nameof(statistics.CountOfDocumentsConflicts));
            writer.WriteInteger(statistics.CountOfDocumentsConflicts);
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.CountOfConflicts));
            writer.WriteInteger(statistics.CountOfConflicts);
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.CountOfAttachments));
            writer.WriteInteger(statistics.CountOfAttachments);
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.CountOfUniqueAttachments));
            writer.WriteInteger(statistics.CountOfUniqueAttachments);
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.DatabaseChangeVector));
            writer.WriteString(statistics.DatabaseChangeVector);
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.DatabaseId));
            writer.WriteString(statistics.DatabaseId);
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.Is64Bit));
            writer.WriteBool(statistics.Is64Bit);
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.Pager));
            writer.WriteString(statistics.Pager);
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.LastDocEtag));
            if (statistics.LastDocEtag.HasValue)
            {
                writer.WriteInteger(statistics.LastDocEtag.Value);
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName((nameof(statistics.DatabaseChangeVector)));
            writer.WriteString(statistics.DatabaseChangeVector);
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.LastIndexingTime));
            if (statistics.LastIndexingTime.HasValue)
            {
                writer.WriteDateTime(statistics.LastIndexingTime.Value, isUtc: true);
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.SizeOnDisk));
            writer.WriteStartObject();

            writer.WritePropertyName(nameof(statistics.SizeOnDisk.HumaneSize));
            writer.WriteString(statistics.SizeOnDisk.HumaneSize);
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.SizeOnDisk.SizeInBytes));
            writer.WriteInteger(statistics.SizeOnDisk.SizeInBytes);

            writer.WriteEndObject();
            writer.WriteComma();

            writer.WritePropertyName(nameof(statistics.Indexes));
            writer.WriteStartArray();
            var isFirstInternal = true;

            foreach (var index in statistics.Indexes)
            {
                if (isFirstInternal == false)
                {
                    writer.WriteComma();
                }

                isFirstInternal = false;

                writer.WriteStartObject();

                writer.WritePropertyName(nameof(index.IsStale));
                writer.WriteBool(index.IsStale);
                writer.WriteComma();

                writer.WritePropertyName(nameof(index.Name));
                writer.WriteString(index.Name);
                writer.WriteComma();

                writer.WritePropertyName(nameof(index.Etag));
                writer.WriteInteger(index.Etag);
                writer.WriteComma();

                writer.WritePropertyName(nameof(index.LockMode));
                writer.WriteString(index.LockMode.ToString());
                writer.WriteComma();

                writer.WritePropertyName(nameof(index.Priority));
                writer.WriteString(index.Priority.ToString());
                writer.WriteComma();

                writer.WritePropertyName(nameof(index.State));
                writer.WriteString(index.State.ToString());
                writer.WriteComma();

                writer.WritePropertyName(nameof(index.Type));
                writer.WriteString(index.Type.ToString());
                writer.WriteComma();

                writer.WritePropertyName(nameof(index.LastIndexingTime));
                if (index.LastIndexingTime.HasValue)
                {
                    writer.WriteDateTime(index.LastIndexingTime.Value, isUtc: true);
                }
                else
                {
                    writer.WriteNull();
                }

                writer.WriteEndObject();
            }
            writer.WriteEndArray();

            writer.WriteEndObject();
        }
Esempio n. 10
0
        public static void WriteTreePagesRecursively(this BlittableJsonTextWriter writer, IEnumerable <ReduceTreePage> pages)
        {
            var first = true;

            foreach (var page in pages)
            {
                if (first == false)
                {
                    writer.WriteComma();
                }

                writer.WriteStartObject();

                writer.WritePropertyName(nameof(TreePage.PageNumber));
                writer.WriteInteger(page.PageNumber);
                writer.WriteComma();

                writer.WritePropertyName(nameof(ReduceTreePage.AggregationResult));
                if (page.AggregationResult != null)
                {
                    writer.WriteObject(page.AggregationResult);
                }
                else
                {
                    writer.WriteNull();
                }
                writer.WriteComma();

                writer.WritePropertyName(nameof(ReduceTreePage.Children));
                if (page.Children != null)
                {
                    writer.WriteStartArray();
                    WriteTreePagesRecursively(writer, page.Children);
                    writer.WriteEndArray();
                }
                else
                {
                    writer.WriteNull();
                }
                writer.WriteComma();

                writer.WritePropertyName(nameof(ReduceTreePage.Entries));
                if (page.Entries != null)
                {
                    writer.WriteStartArray();

                    var firstEntry = true;
                    foreach (var entry in page.Entries)
                    {
                        if (firstEntry == false)
                        {
                            writer.WriteComma();
                        }

                        writer.WriteStartObject();

                        writer.WritePropertyName(nameof(MapResultInLeaf.Data));
                        writer.WriteObject(entry.Data);
                        writer.WriteComma();

                        writer.WritePropertyName(nameof(MapResultInLeaf.Source));
                        writer.WriteString(entry.Source);

                        writer.WriteEndObject();

                        firstEntry = false;
                    }

                    writer.WriteEndArray();
                }
                else
                {
                    writer.WriteNull();
                }

                writer.WriteEndObject();
                first = false;
            }
        }
Esempio n. 11
0
        public static void WriteQueryResult <TResult, TInclude>(this BlittableJsonTextWriter writer, JsonOperationContext context, QueryResultBase <TResult, TInclude> result, bool metadataOnly, out int numberOfResults, bool partial = false)
        {
            if (partial == false)
            {
                writer.WriteStartObject();
            }

            writer.WritePropertyName(nameof(result.IndexName));
            writer.WriteString(result.IndexName);
            writer.WriteComma();

            var results = (object)result.Results;

            if (results is List <Document> documents)
            {
                writer.WritePropertyName(nameof(result.Results));
                writer.WriteDocuments(context, documents, metadataOnly, out numberOfResults);
                writer.WriteComma();
            }
            else if (results is List <BlittableJsonReaderObject> objects)
            {
                writer.WritePropertyName(nameof(result.Results));
                writer.WriteObjects(context, objects, out numberOfResults);
                writer.WriteComma();
            }
            else
            {
                throw new NotSupportedException($"Cannot write query result of '{typeof(TResult)}' type in '{result.GetType()}'.");
            }

            var includes = (object)result.Includes;

            if (includes is List <Document> includeDocuments)
            {
                writer.WritePropertyName(nameof(result.Includes));
                writer.WriteIncludes(context, includeDocuments);
                writer.WriteComma();
            }
            else if (includes is List <BlittableJsonReaderObject> includeObjects)
            {
                if (includeObjects.Count != 0)
                {
                    throw new NotSupportedException("Cannot write query includes of List<BlittableJsonReaderObject>, but got non zero response");
                }

                writer.WritePropertyName(nameof(result.Includes));
                writer.WriteStartObject();
                writer.WriteEndObject();
                writer.WriteComma();
            }
            else
            {
                throw new NotSupportedException($"Cannot write query includes of '{typeof(TInclude)}' type in '{result.GetType()}'.");
            }

            writer.WritePropertyName(nameof(result.IndexTimestamp));
            writer.WriteString(result.IndexTimestamp.ToString(DefaultFormat.DateTimeFormatsToWrite));
            writer.WriteComma();

            writer.WritePropertyName(nameof(result.LastQueryTime));
            writer.WriteString(result.LastQueryTime.ToString(DefaultFormat.DateTimeFormatsToWrite));
            writer.WriteComma();

            writer.WritePropertyName(nameof(result.IsStale));
            writer.WriteBool(result.IsStale);
            writer.WriteComma();

            writer.WritePropertyName(nameof(result.ResultEtag));
            writer.WriteInteger(result.ResultEtag);

            if (partial == false)
            {
                writer.WriteEndObject();
            }
        }
Esempio n. 12
0
 private static void WriteDocumentInternal(this BlittableJsonTextWriter writer, JsonOperationContext context, Document document)
 {
     writer.WriteStartObject();
     WriteDocumentProperties(writer, context, document);
     writer.WriteEndObject();
 }
Esempio n. 13
0
        public static void WriteFacetResult(this BlittableJsonTextWriter writer, JsonOperationContext context, FacetResult result)
        {
            writer.WriteStartObject();

            writer.WritePropertyName(nameof(result.RemainingHits));
            writer.WriteInteger(result.RemainingHits);
            writer.WriteComma();

            writer.WritePropertyName(nameof(result.RemainingTermsCount));
            writer.WriteInteger(result.RemainingTermsCount);
            writer.WriteComma();

            writer.WritePropertyName(nameof(result.RemainingTerms));
            writer.WriteStartArray();
            var isFirstInternal = true;

            foreach (var term in result.RemainingTerms)
            {
                if (isFirstInternal == false)
                {
                    writer.WriteComma();
                }

                isFirstInternal = false;

                writer.WriteString(term);
            }
            writer.WriteEndArray();
            writer.WriteComma();

            writer.WritePropertyName(nameof(result.Values));
            writer.WriteStartArray();
            isFirstInternal = true;
            foreach (var value in result.Values)
            {
                if (isFirstInternal == false)
                {
                    writer.WriteComma();
                }

                isFirstInternal = false;

                writer.WriteStartObject();

                writer.WritePropertyName(nameof(value.Average));
                if (value.Average.HasValue)
                {
                    using (var lazyStringValue = context.GetLazyString(value.Average.ToInvariantString()))
                        writer.WriteDouble(new LazyNumberValue(lazyStringValue));
                }
                else
                {
                    writer.WriteNull();
                }
                writer.WriteComma();

                writer.WritePropertyName(nameof(value.Count));
                if (value.Count.HasValue)
                {
                    writer.WriteInteger(value.Count.Value);
                }
                else
                {
                    writer.WriteNull();
                }
                writer.WriteComma();

                writer.WritePropertyName(nameof(value.Hits));
                writer.WriteInteger(value.Hits);
                writer.WriteComma();

                writer.WritePropertyName(nameof(value.Max));
                if (value.Max.HasValue)
                {
                    using (var lazyStringValue = context.GetLazyString(value.Max.ToInvariantString()))
                        writer.WriteDouble(new LazyNumberValue(lazyStringValue));
                }
                else
                {
                    writer.WriteNull();
                }
                writer.WriteComma();

                writer.WritePropertyName(nameof(value.Min));
                if (value.Min.HasValue)
                {
                    using (var lazyStringValue = context.GetLazyString(value.Min.ToInvariantString()))
                        writer.WriteDouble(new LazyNumberValue(lazyStringValue));
                }
                else
                {
                    writer.WriteNull();
                }
                writer.WriteComma();

                writer.WritePropertyName(nameof(value.Range));
                writer.WriteString(value.Range);
                writer.WriteComma();

                writer.WritePropertyName(nameof(value.Sum));
                if (value.Sum.HasValue)
                {
                    using (var lazyStringValue = context.GetLazyString(value.Sum.ToInvariantString()))
                        writer.WriteDouble(new LazyNumberValue(lazyStringValue));
                }
                else
                {
                    writer.WriteNull();
                }

                writer.WriteEndObject();
            }
            writer.WriteEndArray();

            writer.WriteEndObject();
        }
        public static void WriteIndexQuery(this BlittableJsonTextWriter writer, DocumentConventions conventions, JsonOperationContext context, IndexQuery query)
        {
            writer.WriteStartObject();

            writer.WritePropertyName(nameof(query.Query));
            writer.WriteString(query.Query);
            writer.WriteComma();

            if (query.PageSizeSet && query.PageSize >= 0)
            {
                writer.WritePropertyName(nameof(query.PageSize));
                writer.WriteInteger(query.PageSize);
                writer.WriteComma();
            }

            if (query.WaitForNonStaleResults)
            {
                writer.WritePropertyName(nameof(query.WaitForNonStaleResults));
                writer.WriteBool(query.WaitForNonStaleResults);
                writer.WriteComma();
            }

            if (query.CutoffEtag.HasValue)
            {
                writer.WritePropertyName(nameof(query.CutoffEtag));
                writer.WriteInteger(query.CutoffEtag.Value);
                writer.WriteComma();
            }

            if (query.Start > 0)
            {
                writer.WritePropertyName(nameof(query.Start));
                writer.WriteInteger(query.Start);
                writer.WriteComma();
            }

            if (query.WaitForNonStaleResultsTimeout.HasValue)
            {
                writer.WritePropertyName(nameof(query.WaitForNonStaleResultsTimeout));
                writer.WriteString(query.WaitForNonStaleResultsTimeout.Value.ToInvariantString());
                writer.WriteComma();
            }

            if (query.DisableCaching)
            {
                writer.WritePropertyName(nameof(query.DisableCaching));
                writer.WriteBool(query.DisableCaching);
                writer.WriteComma();
            }

            if (query.ExplainScores)
            {
                writer.WritePropertyName(nameof(query.DisableCaching));
                writer.WriteBool(query.DisableCaching);
                writer.WriteComma();
            }

            if (query.ShowTimings)
            {
                writer.WritePropertyName(nameof(query.ShowTimings));
                writer.WriteBool(query.ShowTimings);
                writer.WriteComma();
            }

            if (query.SkipDuplicateChecking)
            {
                writer.WritePropertyName(nameof(query.SkipDuplicateChecking));
                writer.WriteBool(query.SkipDuplicateChecking);
                writer.WriteComma();
            }

            writer.WritePropertyName(nameof(query.QueryParameters));
            if (query.QueryParameters != null)
            {
                writer.WriteObject(EntityToBlittable.ConvertEntityToBlittable(query.QueryParameters, conventions, context));
            }
            else
            {
                writer.WriteNull();
            }

            writer.WriteEndObject();
        }
Esempio n. 15
0
        private async Task Graph(DocumentsOperationContext context, RequestTimeTracker tracker, HttpMethod method)
        {
            var indexQuery = await GetIndexQuery(context, method, tracker);

            var queryRunner = Database.QueryRunner.GetRunner(indexQuery);

            if (!(queryRunner is GraphQueryRunner gqr))
            {
                throw new InvalidOperationException("The specified query is not a graph query.");
            }

            using (var token = CreateTimeLimitedQueryToken())
                using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
                {
                    var results = await gqr.GetAnalyzedQueryResults(indexQuery, ctx, null, token);

                    var nodes  = new DynamicJsonArray();
                    var edges  = new DynamicJsonArray();
                    var output = new DynamicJsonValue
                    {
                        ["Nodes"] = nodes,
                        ["Edges"] = edges
                    };

                    foreach (var item in results.Nodes)
                    {
                        var val = item.Value;
                        if (val is Document d)
                        {
                            d.EnsureMetadata();
                            val = d.Data;
                        }
                        nodes.Add(new DynamicJsonValue
                        {
                            ["Id"]    = item.Key,
                            ["Value"] = val
                        });
                    }

                    foreach (var edge in results.Edges)
                    {
                        var array = new DynamicJsonArray();
                        var djv   = new DynamicJsonValue
                        {
                            ["Name"]    = edge.Key,
                            ["Results"] = array
                        };
                        foreach (var item in edge.Value)
                        {
                            var edgeVal = item.Edge;
                            if (edgeVal is Document d)
                            {
                                edgeVal = d.Id?.ToString() ?? "anonymous/" + Guid.NewGuid();
                            }
                            array.Add(new DynamicJsonValue
                            {
                                ["From"] = item.Source,
                                ["To"]   = item.Destination,
                                ["Edge"] = edgeVal
                            });
                        }
                        edges.Add(djv);
                    }

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        context.Write(writer, output);
                    }
                }
        }
Esempio n. 16
0
        public static void WriteIndexDefinition(this BlittableJsonTextWriter writer, JsonOperationContext context, IndexDefinition indexDefinition, bool removeAnalyzers = false)
        {
            writer.WriteStartObject();

            writer.WritePropertyName(nameof(indexDefinition.Name));
            writer.WriteString(indexDefinition.Name);
            writer.WriteComma();

            writer.WritePropertyName(nameof(indexDefinition.Etag));
            writer.WriteInteger(indexDefinition.Etag);
            writer.WriteComma();

            writer.WritePropertyName(nameof(indexDefinition.Type));
            writer.WriteString(indexDefinition.Type.ToString());
            writer.WriteComma();

            writer.WritePropertyName(nameof(indexDefinition.LockMode));
            if (indexDefinition.LockMode.HasValue)
            {
                writer.WriteString(indexDefinition.LockMode.ToString());
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName(nameof(indexDefinition.Priority));
            if (indexDefinition.Priority.HasValue)
            {
                writer.WriteString(indexDefinition.Priority.ToString());
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName(nameof(indexDefinition.OutputReduceToCollection));
            writer.WriteString(indexDefinition.OutputReduceToCollection);
            writer.WriteComma();

            writer.WritePropertyName(nameof(indexDefinition.Configuration));
            writer.WriteStartObject();
            var isFirstInternal = true;

            foreach (var kvp in indexDefinition.Configuration)
            {
                if (isFirstInternal == false)
                {
                    writer.WriteComma();
                }

                isFirstInternal = false;

                writer.WritePropertyName(kvp.Key);
                writer.WriteString(kvp.Value);
            }
            writer.WriteEndObject();
            writer.WriteComma();

            writer.WritePropertyName(nameof(indexDefinition.AdditionalSources));
            writer.WriteStartObject();
            isFirstInternal = true;
            foreach (var kvp in indexDefinition.AdditionalSources)
            {
                if (isFirstInternal == false)
                {
                    writer.WriteComma();
                }

                isFirstInternal = false;

                writer.WritePropertyName(kvp.Key);
                writer.WriteString(kvp.Value);
            }
            writer.WriteEndObject();
            writer.WriteComma();

            writer.WritePropertyName(nameof(indexDefinition.IsTestIndex));
            writer.WriteBool(indexDefinition.IsTestIndex);
            writer.WriteComma();

            writer.WritePropertyName(nameof(indexDefinition.Reduce));
            if (string.IsNullOrWhiteSpace(indexDefinition.Reduce) == false)
            {
                writer.WriteString(indexDefinition.Reduce);
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName(nameof(indexDefinition.Maps));
            writer.WriteStartArray();
            isFirstInternal = true;
            foreach (var map in indexDefinition.Maps)
            {
                if (isFirstInternal == false)
                {
                    writer.WriteComma();
                }

                isFirstInternal = false;
                writer.WriteString(map);
            }
            writer.WriteEndArray();
            writer.WriteComma();

            writer.WritePropertyName(nameof(indexDefinition.Fields));
            writer.WriteStartObject();
            isFirstInternal = true;
            foreach (var kvp in indexDefinition.Fields)
            {
                if (isFirstInternal == false)
                {
                    writer.WriteComma();
                }

                isFirstInternal = false;
                writer.WritePropertyName(kvp.Key);
                if (kvp.Value != null)
                {
                    writer.WriteIndexFieldOptions(context, kvp.Value, removeAnalyzers);
                }
                else
                {
                    writer.WriteNull();
                }
            }
            writer.WriteEndObject();

            writer.WriteEndObject();
        }
Esempio n. 17
0
            private void WriteTablesArray(string name, List <SqlMigrationTable> tables, BlittableJsonTextWriter writer)
            {
                writer.WritePropertyName(name);
                writer.WriteStartArray();

                var first = true;

                if (tables != null)
                {
                    foreach (var table in tables)
                    {
                        if (first)
                        {
                            first = false;
                        }
                        else
                        {
                            writer.WriteComma();
                        }

                        writer.WriteStartObject();

                        writer.WritePropertyName(nameof(table.Name));
                        writer.WriteString(table.Name);
                        writer.WriteComma();

                        writer.WritePropertyName(nameof(table.Query));
                        writer.WriteString(table.Query);
                        writer.WriteComma();

                        writer.WritePropertyName(nameof(table.Patch));
                        writer.WriteString(table.Patch);
                        writer.WriteComma();

                        writer.WritePropertyName(nameof(table.NewName));
                        writer.WriteString(table.NewName);
                        writer.WriteComma();

                        WriteTablesArray(nameof(table.EmbeddedTables), table.EmbeddedTables, writer);

                        writer.WriteEndObject();
                    }
                }

                writer.WriteEndArray();
            }
Esempio n. 18
0
        public static void WriteIndexProgress(this BlittableJsonTextWriter writer, JsonOperationContext context, IndexProgress progress)
        {
            writer.WriteStartObject();

            writer.WritePropertyName(nameof(progress.IsStale));
            writer.WriteBool(progress.IsStale);
            writer.WriteComma();

            writer.WritePropertyName(nameof(progress.Collections));
            if (progress.Collections != null)
            {
                writer.WriteStartObject();
                var isFirst = true;
                foreach (var kvp in progress.Collections)
                {
                    if (isFirst == false)
                    {
                        writer.WriteComma();
                    }

                    isFirst = false;

                    writer.WritePropertyName(kvp.Key);

                    writer.WriteStartObject();

                    writer.WritePropertyName(nameof(kvp.Value.LastProcessedDocumentEtag));
                    writer.WriteInteger(kvp.Value.LastProcessedDocumentEtag);
                    writer.WriteComma();

                    writer.WritePropertyName(nameof(kvp.Value.LastProcessedTombstoneEtag));
                    writer.WriteInteger(kvp.Value.LastProcessedTombstoneEtag);
                    writer.WriteComma();

                    writer.WritePropertyName(nameof(kvp.Value.NumberOfDocumentsToProcess));
                    writer.WriteInteger(kvp.Value.NumberOfDocumentsToProcess);
                    writer.WriteComma();

                    writer.WritePropertyName(nameof(kvp.Value.NumberOfTombstonesToProcess));
                    writer.WriteInteger(kvp.Value.NumberOfTombstonesToProcess);
                    writer.WriteComma();

                    writer.WritePropertyName(nameof(kvp.Value.TotalNumberOfDocuments));
                    writer.WriteInteger(kvp.Value.TotalNumberOfDocuments);
                    writer.WriteComma();

                    writer.WritePropertyName(nameof(kvp.Value.TotalNumberOfTombstones));
                    writer.WriteInteger(kvp.Value.TotalNumberOfTombstones);

                    writer.WriteEndObject();
                }
                writer.WriteEndObject();
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName(nameof(progress.Name));
            writer.WriteString(progress.Name);
            writer.WriteComma();

            writer.WritePropertyName(nameof(progress.Type));
            writer.WriteString(progress.Type.ToString());
            writer.WriteComma();

            writer.WritePropertyName(nameof(progress.Etag));
            writer.WriteInteger(progress.Etag);

            writer.WriteEndObject();
        }
Esempio n. 19
0
        public async Task PostImportAsync()
        {
            DocumentsOperationContext context;

            using (ContextPool.AllocateOperationContext(out context))
            {
                if (HttpContext.Request.HasFormContentType == false)
                {
                    HttpContext.Response.StatusCode = 400; // Bad request
                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        context.Write(writer, new DynamicJsonValue
                        {
                            ["Type"]  = "Error",
                            ["Error"] = "This endpoint requires form content type"
                        });
                        return;
                    }
                }

                var operationId = GetLongQueryString("operationId", required: true) ?? -1;
                var token       = CreateOperationToken();
                var sp          = Stopwatch.StartNew();

                var result = new ImportResult();
                await Database.Operations.AddOperation("Import to: " + Database.Name,
                                                       DatabaseOperations.PendingOperationType.DatabaseImport,
                                                       onProgress =>
                {
                    return(Task.Run(async() =>
                    {
                        try
                        {
                            var boundary = MultipartRequestHelper.GetBoundary(
                                MediaTypeHeaderValue.Parse(HttpContext.Request.ContentType),
                                MultipartRequestHelper.MultipartBoundaryLengthLimit);
                            var reader = new MultipartReader(boundary, HttpContext.Request.Body);
                            DatabaseSmugglerOptions smugglerOptions = null;

                            while (true)
                            {
                                var section = await reader.ReadNextSectionAsync().ConfigureAwait(false);
                                if (section == null)
                                {
                                    break;
                                }

                                ContentDispositionHeaderValue contentDisposition;
                                var hasContentDispositionHeader = ContentDispositionHeaderValue.TryParse(section.ContentDisposition,
                                                                                                         out contentDisposition);

                                if (hasContentDispositionHeader == false)
                                {
                                    continue;
                                }

                                if (MultipartRequestHelper.HasFormDataContentDisposition(contentDisposition))
                                {
                                    var key = HeaderUtilities.RemoveQuotes(contentDisposition.Name);
                                    if (key != "importOptions")
                                    {
                                        continue;
                                    }

                                    var blittableJson = await context.ReadForMemoryAsync(section.Body, "importOptions");
                                    smugglerOptions = JsonDeserializationServer.DatabaseSmugglerOptions(blittableJson);
                                    continue;
                                }

                                if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition) == false)
                                {
                                    continue;
                                }

                                var stream = new GZipStream(section.Body, CompressionMode.Decompress);
                                var fileImport =
                                    await DoImportInternal(context, stream, smugglerOptions, onProgress).ConfigureAwait(false);
                                result.DocumentsCount += fileImport.DocumentsCount;
                                result.Warnings.AddRange(fileImport.Warnings);
                                result.IdentitiesCount += fileImport.IdentitiesCount;
                                result.Message = fileImport.Message;
                                result.RevisionDocumentsCount += fileImport.RevisionDocumentsCount;
                                result.IndexesCount += fileImport.IndexesCount;
                                result.TransformersCount += fileImport.TransformersCount;
                            }
                        }
                        catch (Exception e)
                        {
                            result.Message = $"Error occured during export. Exception: {e.Message}";
                            result.Exception = e.ToString();
                        }

                        return (IOperationResult)result;
                    }));
                }, operationId, token).ConfigureAwait(false);

                WriteImportResult(context, sp, result, ResponseBodyStream());
            }
        }
Esempio n. 20
0
        public static void WriteIndexStats(this BlittableJsonTextWriter writer, JsonOperationContext context, IndexStats stats)
        {
            var djv = (DynamicJsonValue)TypeConverter.ToBlittableSupportedType(stats);

            writer.WriteObject(context.ReadObject(djv, "index/stats"));
        }
Esempio n. 21
0
        public Task GetClusterTopology()
        {
            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (context.OpenReadTransaction())
                {
                    var topology = ServerStore.GetClusterTopology(context);
                    var nodeTag  = ServerStore.NodeTag;

                    if (topology.AllNodes.Count == 0)
                    {
                        var tag       = ServerStore.NodeTag ?? "A";
                        var serverUrl = ServerStore.GetNodeHttpServerUrl(HttpContext.Request.GetClientRequestedNodeUrl());

                        topology = new ClusterTopology(
                            topology.TopologyId ?? "dummy",
                            new Dictionary <string, string>
                        {
                            [tag] = serverUrl
                        },
                            new Dictionary <string, string>(),
                            new Dictionary <string, string>(),
                            tag
                            );
                        nodeTag = tag;
                    }
                    else
                    {
                        var isClientIndependent = GetBoolValueQueryString("clientIndependent", false) ?? false;
                        if (isClientIndependent == false)
                        {
                            topology.ReplaceCurrentNodeUrlWithClientRequestedNodeUrlIfNecessary(ServerStore, HttpContext);
                        }
                    }

                    HttpContext.Response.StatusCode = (int)HttpStatusCode.OK;

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        var loadLicenseLimits  = ServerStore.LoadLicenseLimits();
                        var nodeLicenseDetails = loadLicenseLimits == null ?
                                                 null : DynamicJsonValue.Convert(loadLicenseLimits.NodeLicenseDetails);
                        var json = new DynamicJsonValue
                        {
                            ["Topology"]           = topology.ToSortedJson(),
                            ["Leader"]             = ServerStore.LeaderTag,
                            ["LeaderShipDuration"] = ServerStore.Engine.CurrentLeader?.LeaderShipDuration,
                            ["CurrentState"]       = ServerStore.CurrentRachisState,
                            ["NodeTag"]            = nodeTag,
                            ["CurrentTerm"]        = ServerStore.Engine.CurrentTerm,
                            ["NodeLicenseDetails"] = nodeLicenseDetails,
                            [nameof(ServerStore.Engine.LastStateChangeReason)] = ServerStore.LastStateChangeReason()
                        };
                        var clusterErrors = ServerStore.GetClusterErrors();
                        if (clusterErrors.Count > 0)
                        {
                            json["Errors"] = clusterErrors;
                        }

                        var nodesStatues = ServerStore.GetNodesStatuses();
                        json["Status"] = DynamicJsonValue.Convert(nodesStatues);

                        context.Write(writer, json);
                        writer.Flush();
                    }
                }

            return(Task.CompletedTask);
        }
Esempio n. 22
0
        private static void WriteIndexFieldOptions(this BlittableJsonTextWriter writer, JsonOperationContext context, IndexFieldOptions options, bool removeAnalyzers)
        {
            writer.WriteStartObject();

            writer.WritePropertyName(nameof(options.Analyzer));
            if (string.IsNullOrWhiteSpace(options.Analyzer) == false && !removeAnalyzers)
            {
                writer.WriteString(options.Analyzer);
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName(nameof(options.Indexing));
            if (options.Indexing.HasValue)
            {
                writer.WriteString(options.Indexing.ToString());
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName(nameof(options.Storage));
            if (options.Storage.HasValue)
            {
                writer.WriteString(options.Storage.ToString());
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName(nameof(options.Suggestions));
            if (options.Suggestions.HasValue)
            {
                writer.WriteBool(options.Suggestions.Value);
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName(nameof(options.TermVector));
            if (options.TermVector.HasValue)
            {
                writer.WriteString(options.TermVector.ToString());
            }
            else
            {
                writer.WriteNull();
            }
            writer.WriteComma();

            writer.WritePropertyName(nameof(options.Spatial));
            if (options.Spatial != null)
            {
                writer.WriteStartObject();

                writer.WritePropertyName(nameof(options.Spatial.Type));
                writer.WriteString(options.Spatial.Type.ToString());
                writer.WriteComma();

                writer.WritePropertyName(nameof(options.Spatial.MaxTreeLevel));
                writer.WriteInteger(options.Spatial.MaxTreeLevel);
                writer.WriteComma();

                writer.WritePropertyName(nameof(options.Spatial.MaxX));
                LazyStringValue lazyStringValue;
                using (lazyStringValue = context.GetLazyString(CharExtensions.ToInvariantString(options.Spatial.MaxX)))
                    writer.WriteDouble(new LazyNumberValue(lazyStringValue));
                writer.WriteComma();

                writer.WritePropertyName(nameof(options.Spatial.MaxY));
                using (lazyStringValue = context.GetLazyString(CharExtensions.ToInvariantString(options.Spatial.MaxY)))
                    writer.WriteDouble(new LazyNumberValue(lazyStringValue));
                writer.WriteComma();

                writer.WritePropertyName(nameof(options.Spatial.MinX));
                using (lazyStringValue = context.GetLazyString(CharExtensions.ToInvariantString(options.Spatial.MinX)))
                    writer.WriteDouble(new LazyNumberValue(lazyStringValue));
                writer.WriteComma();

                writer.WritePropertyName(nameof(options.Spatial.MinY));
                using (lazyStringValue = context.GetLazyString(CharExtensions.ToInvariantString(options.Spatial.MinY)))
                    writer.WriteDouble(new LazyNumberValue(lazyStringValue));
                writer.WriteComma();

                writer.WritePropertyName(nameof(options.Spatial.Strategy));
                writer.WriteString(options.Spatial.Strategy.ToString());
                writer.WriteComma();

                writer.WritePropertyName(nameof(options.Spatial.Units));
                writer.WriteString(options.Spatial.Units.ToString());

                writer.WriteEndObject();
            }
            else
            {
                writer.WriteNull();
            }

            writer.WriteEndObject();
        }
Esempio n. 23
0
        public async Task Patch()
        {
            var id = GetQueryStringValueAndAssertIfSingleAndNotEmpty("id");

            var isTest    = GetBoolValueQueryString("test", required: false) ?? false;
            var debugMode = GetBoolValueQueryString("debug", required: false) ?? isTest;
            var skipPatchIfChangeVectorMismatch = GetBoolValueQueryString("skipPatchIfChangeVectorMismatch", required: false) ?? false;

            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
            {
                var request = context.Read(RequestBodyStream(), "ScriptedPatchRequest");
                if (request.TryGet("Patch", out BlittableJsonReaderObject patchCmd) == false || patchCmd == null)
                {
                    throw new ArgumentException("The 'Patch' field in the body request is mandatory");
                }

                var patch = PatchRequest.Parse(patchCmd, out var patchArgs);

                PatchRequest patchIfMissing = null;
                BlittableJsonReaderObject patchIfMissingArgs = null;
                if (request.TryGet("PatchIfMissing", out BlittableJsonReaderObject patchIfMissingCmd) && patchIfMissingCmd != null)
                {
                    patchIfMissing = PatchRequest.Parse(patchIfMissingCmd, out patchIfMissingArgs);
                }

                var changeVector = context.GetLazyString(GetStringFromHeaders("If-Match"));

                using (var command = new PatchDocumentCommand(context,
                                                              id,
                                                              changeVector,
                                                              skipPatchIfChangeVectorMismatch,
                                                              (patch, patchArgs),
                                                              (patchIfMissing, patchIfMissingArgs),
                                                              Database,
                                                              isTest,
                                                              debugMode,
                                                              true
                                                              ))
                {
                    if (isTest == false)
                    {
                        await Database.TxMerger.Enqueue(command);
                    }
                    else
                    {
                        // PutDocument requires the write access to the docs storage
                        // testing patching is rare enough not to optimize it
                        using (context.OpenWriteTransaction())
                        {
                            command.Execute(context, null);
                        }
                    }

                    switch (command.PatchResult.Status)
                    {
                    case PatchStatus.DocumentDoesNotExist:
                        HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
                        return;

                    case PatchStatus.Created:
                        HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
                        break;

                    case PatchStatus.Skipped:
                        HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
                        return;

                    case PatchStatus.Patched:
                    case PatchStatus.NotModified:
                        HttpContext.Response.StatusCode = (int)HttpStatusCode.OK;
                        break;

                    default:
                        throw new ArgumentOutOfRangeException();
                    }

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        writer.WriteStartObject();

                        writer.WritePropertyName(nameof(command.PatchResult.Status));
                        writer.WriteString(command.PatchResult.Status.ToString());
                        writer.WriteComma();

                        writer.WritePropertyName(nameof(command.PatchResult.ModifiedDocument));
                        writer.WriteObject(command.PatchResult.ModifiedDocument);

                        if (debugMode)
                        {
                            writer.WriteComma();
                            writer.WritePropertyName(nameof(command.PatchResult.OriginalDocument));
                            if (isTest)
                            {
                                writer.WriteObject(command.PatchResult.OriginalDocument);
                            }
                            else
                            {
                                writer.WriteNull();
                            }

                            writer.WriteComma();

                            writer.WritePropertyName(nameof(command.PatchResult.Debug));

                            context.Write(writer, new DynamicJsonValue
                            {
                                ["Info"]    = new DynamicJsonArray(command.DebugOutput),
                                ["Actions"] = command.DebugActions?.GetDebugActions()
                            });
                        }


                        writer.WriteEndObject();
                    }
                }
            }
        }
        public Task GetCollectionFields()
        {
            var collection = GetStringQueryString("collection", required: false);
            var prefix     = GetStringQueryString("prefix", required: false);

            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                using (context.OpenReadTransaction())
                {
                    long   totalResults;
                    string changeVector;
                    string etag = null;

                    if (string.IsNullOrEmpty(collection))
                    {
                        changeVector = DocumentsStorage.GetDatabaseChangeVector(context);
                        totalResults = Database.DocumentsStorage.GetNumberOfDocuments(context);
                        etag         = $"{changeVector}/{totalResults}";
                    }
                    else
                    {
                        changeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context, collection);
                        totalResults = Database.DocumentsStorage.GetCollection(collection, context).Count;

                        if (changeVector != null)
                        {
                            etag = $"{changeVector}/{totalResults}";
                        }
                    }

                    if (etag != null && GetStringFromHeaders("If-None-Match") == etag)
                    {
                        HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
                        return(Task.CompletedTask);
                    }
                    HttpContext.Response.Headers["ETag"] = "\"" + etag + "\"";

                    if (_buffers == null)
                    {
                        _buffers = new BlittableJsonReaderObject.PropertiesInsertionBuffer();
                    }

                    var fields = new Dictionary <LazyStringValue, FieldType>(LazyStringValueComparer.Instance);

                    if (string.IsNullOrEmpty(collection))
                    {
                        foreach (var collectionStats in Database.DocumentsStorage.GetCollections(context))
                        {
                            FetchFieldsForCollection(context, collectionStats.Name, prefix, fields, _buffers);
                        }
                    }
                    else
                    {
                        FetchFieldsForCollection(context, collection, prefix, fields, _buffers);
                    }

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        writer.WriteStartObject();

                        var first = true;
                        foreach (var field in fields)
                        {
                            if (first == false)
                            {
                                writer.WriteComma();
                            }
                            first = false;

                            writer.WritePropertyName(field.Key);
                            writer.WriteString(field.Value.ToString());
                        }

                        writer.WriteEndObject();
                    }

                    return(Task.CompletedTask);
                }
        }
Esempio n. 25
0
        public Task GetAll()
        {
            var start    = GetStart();
            var pageSize = GetPageSize();
            var history  = GetBoolValueQueryString("history", required: false) ?? false;
            var running  = GetBoolValueQueryString("running", required: false) ?? false;
            var id       = GetLongQueryString("id", required: false);
            var name     = GetStringQueryString("name", required: false);

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (context.OpenReadTransaction())
                {
                    IEnumerable <SubscriptionStorage.SubscriptionGeneralDataAndStats> subscriptions;
                    if (string.IsNullOrEmpty(name) && id == null)
                    {
                        subscriptions = running
                        ? Database.SubscriptionStorage.GetAllRunningSubscriptions(context, history, start, pageSize)
                        : Database.SubscriptionStorage.GetAllSubscriptions(context, history, start, pageSize);
                    }
                    else
                    {
                        var subscription = running
                        ? Database
                                           .SubscriptionStorage
                                           .GetRunningSubscription(context, id, name, history)
                        : Database
                                           .SubscriptionStorage
                                           .GetSubscription(context, id, name, history);

                        if (subscription == null)
                        {
                            HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
                            return(Task.CompletedTask);
                        }

                        subscriptions = new[] { subscription };
                    }

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        writer.WriteStartObject();

                        var subscriptionsAsBlittable = subscriptions.Select(x => new DynamicJsonValue()
                        {
                            [nameof(SubscriptionState.SubscriptionId)]   = x.SubscriptionId,
                            [nameof(SubscriptionState.SubscriptionName)] = x.SubscriptionName,
                            [nameof(SubscriptionState.ChangeVectorForNextBatchStartingPoint)] = x.ChangeVectorForNextBatchStartingPoint,
                            [nameof(SubscriptionState.Query)]    = x.Query,
                            [nameof(SubscriptionState.Disabled)] = x.Disabled,
                            [nameof(SubscriptionState.LastClientConnectionTime)] = x.LastClientConnectionTime,
                            [nameof(SubscriptionState.LastBatchAckTime)]         = x.LastBatchAckTime,
                            ["Connection"]        = GetSubscriptionConnectionDJV(x.Connection),
                            ["RecentConnections"] = x.RecentConnections?.Select(r => new DynamicJsonValue()
                            {
                                ["State"] = new DynamicJsonValue()
                                {
                                    ["LatestChangeVectorClientACKnowledged"] = r.SubscriptionState.ChangeVectorForNextBatchStartingPoint,
                                    ["Query"] = r.SubscriptionState.Query
                                },
                                ["Connection"] = GetSubscriptionConnectionDJV(r)
                            }),
                            ["FailedConnections"] = x.RecentRejectedConnections?.Select(r => new DynamicJsonValue()
                            {
                                ["State"] = new DynamicJsonValue()
                                {
                                    ["LatestChangeVectorClientACKnowledged"] = r.SubscriptionState.ChangeVectorForNextBatchStartingPoint,
                                    ["Query"] = r.SubscriptionState.Query
                                },
                                ["Connection"] = GetSubscriptionConnectionDJV(r)
                            }).ToList()
                        });

                        writer.WriteArray(context, "Results", subscriptionsAsBlittable, (w, c, subscription) =>
                        {
                            c.Write(w, subscription);
                        });

                        writer.WriteEndObject();
                    }
                }

            return(Task.CompletedTask);
        }
 public GraphQueryDetailedReporter(BlittableJsonTextWriter writer, DocumentsOperationContext ctx)
 {
     _writer = writer;
     _ctx    = ctx;
 }
 public StreamJsonDocumentQueryResultWriter(HttpResponse response, Stream stream, JsonOperationContext context)
 {
     _context  = context;
     _writer   = new BlittableJsonTextWriter(context, stream);
     _response = response;
 }
Esempio n. 28
0
        private async Task RequestHandler(HttpContext context)
        {
            var requestHandlerContext = new RequestHandlerContext
            {
                HttpContext = context
            };
            Exception exception = null;
            Stopwatch sp        = null;

            try
            {
                context.Response.StatusCode = (int)HttpStatusCode.OK;
                context.Response.Headers["Content-Type"] = "application/json; charset=utf-8";
                context.Response.Headers[Constants.Headers.ServerVersion] = RavenVersionAttribute.Instance.AssemblyVersion;

                if (_server.ServerStore.Initialized == false)
                {
                    await _server.ServerStore.InitializationCompleted.WaitAsync();
                }

                sp = Stopwatch.StartNew();
                await _router.HandlePath(requestHandlerContext);

                sp.Stop();
            }
            catch (Exception e)
            {
                sp?.Stop();
                exception = e;

                CheckVersionAndWrapException(context, ref e);

                MaybeSetExceptionStatusCode(context, _server.ServerStore, e);

                if (context.RequestAborted.IsCancellationRequested)
                {
                    return;
                }

                using (_server.ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext ctx))
                {
                    var djv = new DynamicJsonValue
                    {
                        [nameof(ExceptionDispatcher.ExceptionSchema.Url)]     = $"{context.Request.Path}{context.Request.QueryString}",
                        [nameof(ExceptionDispatcher.ExceptionSchema.Type)]    = e.GetType().FullName,
                        [nameof(ExceptionDispatcher.ExceptionSchema.Message)] = e.Message,
                        [nameof(ExceptionDispatcher.ExceptionSchema.Error)]   = e.ToString()
                    };

#if EXCEPTION_ERROR_HUNT
                    var f = Guid.NewGuid() + ".error";
                    File.WriteAllText(f,
                                      $"{context.Request.Path}{context.Request.QueryString}" + Environment.NewLine + errorString);
#endif

                    MaybeAddAdditionalExceptionData(djv, e);

                    using (var writer = new BlittableJsonTextWriter(ctx, context.Response.Body))
                    {
                        var json = ctx.ReadObject(djv, "exception");
                        writer.WriteObject(json);
                    }

#if EXCEPTION_ERROR_HUNT
                    File.Delete(f);
#endif
                }
            }
            finally
            {
                // check if TW has clients
                if (TrafficWatchManager.HasRegisteredClients)
                {
                    var database = requestHandlerContext.Database?.Name;
                    LogTrafficWatch(context, sp?.ElapsedMilliseconds ?? 0, database);
                }

                if (sp != null && requestHandlerContext.HttpContext.Response.StatusCode != (int)HttpStatusCode.SwitchingProtocols) // exclude web sockets
                {
                    var requestDuration = sp.ElapsedMilliseconds;
                    requestHandlerContext.RavenServer?.Metrics.Requests.UpdateDuration(requestDuration);
                    requestHandlerContext.Database?.Metrics.Requests.UpdateDuration(requestDuration);
                }

                if (_logger.IsInfoEnabled && SkipHttpLogging == false)
                {
                    _logger.Info($"{context.Request.Method} {context.Request.Path.Value}{context.Request.QueryString.Value} - {context.Response.StatusCode} - {(sp?.ElapsedMilliseconds ?? 0):#,#;;0} ms", exception);
                }
            }
        }
        public Task PreviewCollection()
        {
            var start        = GetStart();
            var pageSize     = GetPageSize();
            var collection   = GetStringQueryString("collection", required: false);
            var bindings     = GetStringValuesQueryString("binding", required: false);
            var fullBindings = GetStringValuesQueryString("fullBinding", required: false);

            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                using (context.OpenReadTransaction())
                {
                    Document[] documents;
                    HashSet <LazyStringValue> availableColumns;
                    HashSet <string>          propertiesPreviewToSend;
                    HashSet <string>          fullPropertiesToSend = new HashSet <string>(fullBindings);

                    long   totalResults;
                    string changeVector;
                    string etag = null;

                    if (string.IsNullOrEmpty(collection))
                    {
                        changeVector = DocumentsStorage.GetDatabaseChangeVector(context);
                        totalResults = Database.DocumentsStorage.GetNumberOfDocuments(context);
                        etag         = $"{changeVector}/{totalResults}";
                    }
                    else
                    {
                        changeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context.Transaction.InnerTransaction, context, collection);
                        totalResults = Database.DocumentsStorage.GetCollection(collection, context).Count;

                        if (changeVector != null)
                        {
                            etag = $"{changeVector}/{totalResults}";
                        }
                    }

                    if (etag != null && GetStringFromHeaders("If-None-Match") == etag)
                    {
                        HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
                        return(Task.CompletedTask);
                    }

                    HttpContext.Response.Headers["ETag"] = "\"" + etag + "\"";

                    if (string.IsNullOrEmpty(collection))
                    {
                        documents               = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, start, pageSize).ToArray();
                        availableColumns        = ExtractColumnNames(documents, context);
                        propertiesPreviewToSend = bindings.Count > 0 ? new HashSet <string>(bindings) : new HashSet <string>();
                    }
                    else
                    {
                        documents               = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, collection, start, pageSize).ToArray();
                        availableColumns        = ExtractColumnNames(documents, context);
                        propertiesPreviewToSend = bindings.Count > 0 ? new HashSet <string>(bindings) : availableColumns.Take(ColumnsSamplingLimit).Select(x => x.ToString()).ToHashSet();
                    }

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        writer.WriteStartObject();
                        writer.WritePropertyName("Results");

                        writer.WriteStartArray();

                        var first = true;
                        foreach (var document in documents)
                        {
                            if (first == false)
                            {
                                writer.WriteComma();
                            }
                            first = false;

                            using (document.Data)
                            {
                                WriteDocument(writer, context, document, propertiesPreviewToSend, fullPropertiesToSend);
                            }
                        }

                        writer.WriteEndArray();

                        writer.WriteComma();

                        writer.WritePropertyName("TotalResults");
                        writer.WriteInteger(totalResults);

                        writer.WriteComma();

                        writer.WriteArray("AvailableColumns", availableColumns);

                        writer.WriteEndObject();
                    }

                    return(Task.CompletedTask);
                }
        }
Esempio n. 30
0
        public async Task Create()
        {
            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                using (context.OpenReadTransaction())
                {
                    var json = await context.ReadForMemoryAsync(RequestBodyStream(), null);

                    var options = JsonDeserializationServer.SubscriptionCreationParams(json);

                    if (TrafficWatchManager.HasRegisteredClients)
                    {
                        AddStringToHttpContext(json.ToString(), TrafficWatchChangeType.Subscriptions);
                    }

                    var sub = SubscriptionConnection.ParseSubscriptionQuery(options.Query);

                    if (Enum.TryParse(
                            options.ChangeVector,
                            out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
                    {
                        switch (changeVectorSpecialValue)
                        {
                        case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:

                            options.ChangeVector = null;
                            break;

                        case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
                            options.ChangeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context, sub.Collection);
                            break;
                        }
                    }
                    var id             = GetLongQueryString("id", required: false);
                    var disabled       = GetBoolValueQueryString("disabled", required: false);
                    var mentor         = options.MentorNode;
                    var subscriptionId = await Database.SubscriptionStorage.PutSubscription(options, id, disabled, mentor : mentor);

                    var name = options.Name ?? subscriptionId.ToString();

                    using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverContext))
                        using (serverContext.OpenReadTransaction())
                        {
                            // need to wait on the relevant remote node
                            var node = Database.SubscriptionStorage.GetResponsibleNode(serverContext, name);
                            if (node != null && node != ServerStore.NodeTag)
                            {
                                await WaitForExecutionOnSpecificNode(serverContext, ServerStore.GetClusterTopology(serverContext), node, subscriptionId);
                            }
                        }

                    HttpContext.Response.StatusCode = (int)HttpStatusCode.Created; // Created

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        context.Write(writer, new DynamicJsonValue
                        {
                            ["Name"] = name
                        });
                    }
                }
        }