Esempio n. 1
0
        public IEnumerable <(Document Doc, Exception Exception)> GetDataToSend(
            DocumentsOperationContext docsContext,
            string collection,
            bool revisions,
            SubscriptionState subscription,
            SubscriptionPatchDocument patch,
            long startEtag)
        {
            if (string.IsNullOrEmpty(collection))
            {
                throw new ArgumentException("The collection name must be specified");
            }

            if (revisions)
            {
                if (_db.DocumentsStorage.RevisionsStorage.Configuration == null ||
                    _db.DocumentsStorage.RevisionsStorage.GetRevisionsConfiguration(collection).Active == false)
                {
                    throw new SubscriptionInvalidStateException($"Cannot use a revisions subscription, database {_db.Name} does not have revisions configuration.");
                }

                return(GetRevisionsToSend(docsContext, collection, subscription, startEtag, patch));
            }


            return(GetDocumentsToSend(docsContext, collection, subscription, startEtag, patch));
        }
        private SubscriptionPatchDocument SetupFilterScript()
        {
            SubscriptionPatchDocument patch = null;

            if (string.IsNullOrWhiteSpace(Script) == false)
            {
                patch = new SubscriptionPatchDocument(Script, Functions);
            }
            return(patch);
        }
Esempio n. 3
0
        private SubscriptionPatchDocument SetupFilterScript(SubscriptionCriteria criteria)
        {
            SubscriptionPatchDocument patch = null;

            if (string.IsNullOrWhiteSpace(criteria.FilterJavaScript) == false)
            {
                patch = new SubscriptionPatchDocument(TcpConnection.DocumentDatabase, criteria.FilterJavaScript);
            }
            return(patch);
        }
 public SubscriptionDocumentsFetcher(DocumentDatabase db, int maxBatchSize, long subscriptionId, EndPoint remoteEndpoint, string collection,
                                     bool revisions,
                                     SubscriptionState subscription,
                                     SubscriptionPatchDocument patch)
 {
     _db             = db;
     _logger         = LoggingSource.Instance.GetLogger <SubscriptionDocumentsFetcher>(db.Name);
     _maxBatchSize   = maxBatchSize;
     _subscriptionId = subscriptionId;
     _remoteEndpoint = remoteEndpoint;
     _collection     = collection;
     _revisions      = revisions;
     _subscription   = subscription;
     _patch          = patch;
 }
 public SubscriptionDocumentsFetcher(DocumentDatabase db, int maxBatchSize, long subscriptionId, EndPoint remoteEndpoint, string collection,
                                     bool revisions,
                                     SubscriptionState subscription,
                                     SubscriptionPatchDocument patch)
 {
     _db                   = db;
     _logger               = LoggingSource.Instance.GetLogger <SubscriptionDocumentsFetcher>(db.Name);
     _maxBatchSize         = maxBatchSize;
     _subscriptionId       = subscriptionId;
     _remoteEndpoint       = remoteEndpoint;
     _collection           = collection;
     _revisions            = revisions;
     _subscription         = subscription;
     _patch                = patch;
     _maximumAllowedMemory = new Size((_db.Is32Bits ? 4 : 32) * Voron.Global.Constants.Size.Megabyte, SizeUnit.Bytes);
 }
 public SubscriptionDocumentsFetcher(DocumentDatabase db, int maxBatchSize, long subscriptionId, EndPoint remoteEndpoint, string collection,
                                     bool revisions,
                                     SubscriptionState subscription,
                                     SubscriptionPatchDocument patch)
 {
     _db                   = db;
     _logger               = LoggingSource.Instance.GetLogger <SubscriptionDocumentsFetcher>(db.Name);
     _maxBatchSize         = maxBatchSize;
     _subscriptionId       = subscriptionId;
     _remoteEndpoint       = remoteEndpoint;
     _collection           = collection;
     _revisions            = revisions;
     _subscription         = subscription;
     _patch                = patch;
     _maximumAllowedMemory = new Size((PlatformDetails.Is32Bits ||
                                       _db.Configuration.Storage.ForceUsing32BitsPager) ? 4 : 32, SizeUnit.Megabytes);
 }
Esempio n. 7
0
 private bool DocumentMatchCriteriaScript(SubscriptionPatchDocument patch, DocumentsOperationContext dbContext,
                                          Document doc)
 {
     if (patch == null)
     {
         return(true);
     }
     try
     {
         return(patch.MatchCriteria(dbContext, doc));
     }
     catch (Exception ex)
     {
         if (_logger.IsInfoEnabled)
         {
             _logger.Info(
                 $"Criteria script threw exception for subscription {_options.SubscriptionId} connected to {TcpConnection.TcpClient.Client.RemoteEndPoint} for document id {doc.Key}",
                 ex);
         }
         return(false);
     }
 }
Esempio n. 8
0
        public async Task Try()
        {
            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
            {
                var json = await context.ReadForMemoryAsync(RequestBodyStream(), null);

                var tryout = JsonDeserializationServer.SubscriptionTryout(json);

                var(collection, (script, functions), revisions) = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query);
                SubscriptionPatchDocument patch = null;
                if (string.IsNullOrEmpty(script) == false)
                {
                    patch = new SubscriptionPatchDocument(script, functions);
                }

                if (collection == null)
                {
                    throw new ArgumentException("Collection must be specified");
                }

                var pageSize = GetIntValueQueryString("pageSize") ?? 1;

                var state = new SubscriptionState
                {
                    ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector,
                    Query = tryout.Query
                };

                var fetcher = new SubscriptionDocumentsFetcher(Database, pageSize, -0x42,
                                                               new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), collection, revisions, state, patch);

                if (Enum.TryParse(
                        tryout.ChangeVector,
                        out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
                {
                    switch (changeVectorSpecialValue)
                    {
                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange:
                        state.ChangeVectorForNextBatchStartingPoint = null;
                        break;

                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
                        state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context, collection);
                        break;
                    }
                }

                using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    writer.WriteStartObject();
                    writer.WritePropertyName("Results");
                    writer.WriteStartArray();

                    using (context.OpenReadTransaction())
                    {
                        var first = true;

                        foreach (var itemDetails in fetcher.GetDataToSend(context, 0))
                        {
                            if (itemDetails.Doc.Data == null)
                            {
                                continue;
                            }

                            if (first == false)
                            {
                                writer.WriteComma();
                            }

                            if (itemDetails.Exception == null)
                            {
                                writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false);
                            }
                            else
                            {
                                var docWithExcepton = new DocumentWithException
                                {
                                    Exception    = itemDetails.Exception.ToString(),
                                    ChangeVector = itemDetails.Doc.ChangeVector,
                                    Id           = itemDetails.Doc.Id,
                                    DocumentData = itemDetails.Doc.Data
                                };
                                writer.WriteObject(context.ReadObject(docWithExcepton.ToJson(), ""));
                            }

                            first = false;
                        }
                    }

                    writer.WriteEndArray();
                    writer.WriteEndObject();
                }
            }
        }
Esempio n. 9
0
        private IEnumerable <(Document Doc, Exception Exception)> GetDocumentsToSend(DocumentsOperationContext docsContext,
                                                                                     string collection,
                                                                                     SubscriptionState subscription,
                                                                                     long startEtag, SubscriptionPatchDocument patch)
        {
            int size = 0;

            using (_db.Scripts.GetScriptRunner(patch, true, out var run))
            {
                foreach (var doc in _db.DocumentsStorage.GetDocumentsFrom(
                             docsContext,
                             collection,
                             startEtag + 1,
                             0,
                             int.MaxValue))
                {
                    using (doc.Data)
                    {
                        if (ShouldSendDocument(subscription, run, patch, docsContext, doc, out BlittableJsonReaderObject transformResult, out var exception) == false)
                        {
                            if (exception != null)
                            {
                                yield return(doc, exception);

                                if (++size >= _maxBatchSize)
                                {
                                    yield break;
                                }
                            }
                            else
                            {
                                doc.Data = null;
                                yield return(doc, null);
                            }
                            doc.Data = null;
                        }
Esempio n. 10
0
        public async Task Try()
        {
            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
            {
                BlittableJsonReaderObject json = await context.ReadForMemoryAsync(RequestBodyStream(), null);

                SubscriptionTryout tryout = JsonDeserializationServer.SubscriptionTryout(json);

                SubscriptionConnection.ParsedSubscription sub = SubscriptionConnection.ParseSubscriptionQuery(tryout.Query);
                SubscriptionPatchDocument patch = null;
                if (string.IsNullOrEmpty(sub.Script) == false)
                {
                    patch = new SubscriptionPatchDocument(sub.Script, sub.Functions);
                }

                if (sub.Collection == null)
                {
                    throw new ArgumentException("Collection must be specified");
                }

                const int maxPageSize = 1024;
                var       pageSize    = GetIntValueQueryString("pageSize") ?? 1;
                if (pageSize > maxPageSize)
                {
                    throw new ArgumentException($"Cannot gather more than {maxPageSize} results during tryouts, but requested number was {pageSize}.");
                }

                var state = new SubscriptionState
                {
                    ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector,
                    Query = tryout.Query
                };

                var fetcher = new SubscriptionDocumentsFetcher(Database, int.MaxValue, -0x42,
                                                               new IPEndPoint(HttpContext.Connection.RemoteIpAddress, HttpContext.Connection.RemotePort), sub.Collection, sub.Revisions, state, patch);

                var includeCmd = new IncludeDocumentsCommand(Database.DocumentsStorage, context, sub.Includes, isProjection: patch != null);

                if (Enum.TryParse(
                        tryout.ChangeVector,
                        out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
                {
                    switch (changeVectorSpecialValue)
                    {
                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:
                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange:
                        state.ChangeVectorForNextBatchStartingPoint = null;
                        break;

                    case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
                        using (context.OpenReadTransaction())
                        {
                            state.ChangeVectorForNextBatchStartingPoint = Database.DocumentsStorage.GetLastDocumentChangeVector(context.Transaction.InnerTransaction, context, sub.Collection);
                        }
                        break;
                    }
                }
                else
                {
                    state.ChangeVectorForNextBatchStartingPoint = tryout.ChangeVector;
                }

                var changeVector = state.ChangeVectorForNextBatchStartingPoint.ToChangeVector();
                var cv           = changeVector.FirstOrDefault(x => x.DbId == Database.DbBase64Id);

                var sp        = Stopwatch.StartNew();
                var timeLimit = TimeSpan.FromSeconds(GetIntValueQueryString("timeLimit", false) ?? 15);
                var startEtag = cv.Etag;

                await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
                    using (context.OpenReadTransaction())
                    {
                        writer.WriteStartObject();
                        writer.WritePropertyName("Results");
                        writer.WriteStartArray();
                        var numberOfDocs = 0;
                        while (numberOfDocs == 0 && sp.Elapsed < timeLimit)
                        {
                            var first    = true;
                            var lastEtag = startEtag;
                            foreach (var itemDetails in fetcher.GetDataToSend(context, includeCmd, startEtag))
                            {
                                if (itemDetails.Doc.Data != null)
                                {
                                    using (itemDetails.Doc.Data)
                                    {
                                        includeCmd.Gather(itemDetails.Doc);

                                        if (first == false)
                                        {
                                            writer.WriteComma();
                                        }

                                        if (itemDetails.Exception == null)
                                        {
                                            writer.WriteDocument(context, itemDetails.Doc, metadataOnly: false);
                                        }
                                        else
                                        {
                                            var documentWithException = new DocumentWithException
                                            {
                                                Exception    = itemDetails.Exception.ToString(),
                                                ChangeVector = itemDetails.Doc.ChangeVector,
                                                Id           = itemDetails.Doc.Id,
                                                DocumentData = itemDetails.Doc.Data
                                            };
                                            writer.WriteObject(context.ReadObject(documentWithException.ToJson(), ""));
                                        }

                                        first = false;

                                        if (++numberOfDocs >= pageSize)
                                        {
                                            break;
                                        }
                                    }
                                }

                                if (sp.Elapsed >= timeLimit)
                                {
                                    break;
                                }

                                lastEtag = itemDetails.Doc.Etag;
                            }

                            if (startEtag == lastEtag)
                            {
                                break;
                            }

                            startEtag = lastEtag;
                        }

                        writer.WriteEndArray();
                        writer.WriteComma();
                        writer.WritePropertyName("Includes");
                        var includes = new List <Document>();
                        includeCmd.Fill(includes);
                        await writer.WriteIncludesAsync(context, includes);

                        writer.WriteEndObject();
                    }
            }
        }
Esempio n. 11
0
        private async Task ProcessSubscriptionAsync()
        {
            if (_logger.IsInfoEnabled)
            {
                _logger.Info(
                    $"Starting processing documents for subscription {SubscriptionId} received from {TcpConnection.TcpClient.Client.RemoteEndPoint}");
            }

            using (DisposeOnDisconnect)
                using (RegisterForNotificationOnNewDocuments())
                {
                    var replyFromClientTask = GetReplyFromClientAsync();

                    string subscriptionChangeVectorBeforeCurrentBatch = SubscriptionState.ChangeVectorForNextBatchStartingPoint;

                    _startEtag = GetStartEtagForSubscription(SubscriptionState);
                    _filterAndProjectionScript = SetupFilterAndProjectionScript();
                    _documentsFetcher          = new SubscriptionDocumentsFetcher(TcpConnection.DocumentDatabase, _options.MaxDocsPerBatch, SubscriptionId, TcpConnection.TcpClient.Client.RemoteEndPoint, Collection, Revisions, SubscriptionState, _filterAndProjectionScript);

                    while (CancellationTokenSource.IsCancellationRequested == false)
                    {
                        _buffer.SetLength(0);

                        using (TcpConnection.DocumentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext docsContext))
                        {
                            var sendingCurrentBatchStopwatch = Stopwatch.StartNew();

                            var anyDocumentsSentInCurrentIteration = await TrySendingBatchToClient(docsContext, sendingCurrentBatchStopwatch);

                            if (anyDocumentsSentInCurrentIteration == false)
                            {
                                if (_logger.IsInfoEnabled)
                                {
                                    _logger.Info(
                                        $"Did not find any documents to send for subscription {Options.SubscriptionName}");
                                }
                                await TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed(SubscriptionId,
                                                                                                                   Options.SubscriptionName,
                                                                                                                   _lastChangeVector,
                                                                                                                   subscriptionChangeVectorBeforeCurrentBatch);

                                subscriptionChangeVectorBeforeCurrentBatch = _lastChangeVector;

                                if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
                                {
                                    await SendHeartBeat();
                                }

                                using (docsContext.OpenReadTransaction())
                                {
                                    long globalEtag = TcpConnection.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(docsContext, Collection);

                                    if (globalEtag > _startEtag)
                                    {
                                        continue;
                                    }
                                }

                                if (await WaitForChangedDocuments(replyFromClientTask))
                                {
                                    continue;
                                }
                            }
                        }

                        (replyFromClientTask, subscriptionChangeVectorBeforeCurrentBatch) =
                            await WaitForClientAck(replyFromClientTask, subscriptionChangeVectorBeforeCurrentBatch);
                    }

                    CancellationTokenSource.Token.ThrowIfCancellationRequested();
                }
        }