コード例 #1
0
        private async Task <SubscriptionConnectionClientMessage> GetReplyFromClientAsync()
        {
            try
            {
                using (TcpConnection.ContextPool.AllocateOperationContext(out JsonOperationContext context))
                    using (var reader = await context.ParseToMemoryAsync(
                               TcpConnection.Stream,
                               "Reply from subscription client",
                               BlittableJsonDocumentBuilder.UsageMode.None,
                               TcpConnection.PinnedBuffer))
                    {
                        TcpConnection.RegisterBytesReceived(reader.Size);
                        return(JsonDeserializationServer.SubscriptionConnectionClientMessage(reader));
                    }
            }
            catch (IOException)
            {
                if (_isDisposed == false)
                {
                    throw;
                }

                return(new SubscriptionConnectionClientMessage
                {
                    ChangeVector = null,
                    Type = SubscriptionConnectionClientMessage.MessageType.DisposedNotification
                });
            }
            catch (ObjectDisposedException)
            {
                return(new SubscriptionConnectionClientMessage
                {
                    ChangeVector = null,
                    Type = SubscriptionConnectionClientMessage.MessageType.DisposedNotification
                });
            }
        }
コード例 #2
0
        private void ReadBulkInsert()
        {
            var managedBuffer = new byte[1024 * 32];

            fixed(byte *managedBufferPointer = managedBuffer)
            {
                while (true)
                {
                    // _context.Reset(); - we cannot reset the context here
                    // because the memory is being used by the other threads
                    // we avoid the memory leak of infinite usage by limiting
                    // the number of buffers we get from the context and then
                    // reusing them
                    var len = Read7BitEncodedInt();
                    if (len <= 0)
                    {
                        _docsToWrite.CompleteAdding();
                        break;
                    }

                    BulkInsertDoc buffer;
                    while (true)
                    {
                        bool hasFreeBuffer;
                        try
                        {
                            hasFreeBuffer = _docsToRelease.TryTake(out buffer);
                            if (_docsToRelease.IsAddingCompleted)
                            {
                                if (_logger.IsInfoEnabled)
                                {
                                    _logger.Info("Stopping read bulk insert due to an error in insert documents task");
                                }
                                // error during the insert, just quit and use the error handling to report to the user
                                return;
                            }
                        }
                        catch (Exception ex)
                        {
                            if (_logger.IsInfoEnabled)
                            {
                                _logger.Info("Server internal error while in read documents in bulk insert", ex);
                            }
                            return;
                        }
                        if (hasFreeBuffer == false)
                        {
                            var allocatedMemoryData = TcpConnection.Context.GetMemory(len);
                            buffer = new BulkInsertDoc
                            {
                                Memory  = allocatedMemoryData,
                                Pointer = (byte *)allocatedMemoryData.Address
                            };
                            break;
                        }
                        buffer.Used = 0;
                        if (buffer.Memory.SizeInBytes >= len)
                        {
                            break;
                        }
                    }
                    while (len > 0)
                    {
                        var read = TcpConnection.MultiDocumentParser.Read(managedBuffer, 0, Math.Min(len, managedBuffer.Length));
                        TcpConnection.RegisterBytesReceived(read);
                        if (read == 0)
                        {
                            throw new EndOfStreamException("Could not read expected document");
                        }
                        len -= read;

                        Memory.Copy(buffer.Pointer + buffer.Used, managedBufferPointer, read);

                        buffer.Used += read;
                    }
                    while (_docsToWrite.TryAdd(buffer, 500) == false)
                    {
                        _messagesToClient.Add(ProcessingMessage);
                    }
                }
            }
        }
コード例 #3
0
        private async Task ProcessSubscriptionAysnc()
        {
            if (_logger.IsInfoEnabled)
            {
                _logger.Info($"Starting proccessing documents for subscription {SubscriptionId} received from {TcpConnection.TcpClient.Client.RemoteEndPoint}");
            }

            DocumentsOperationContext dbContext;

            using (DisposeOnDisconnect)
                using (TcpConnection.DocumentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out dbContext))
                {
                    long startEtag;
                    SubscriptionCriteria criteria;
                    TcpConnection.DocumentDatabase.SubscriptionStorage.GetCriteriaAndEtag(_options.SubscriptionId, dbContext,
                                                                                          out criteria, out startEtag);

                    var replyFromClientTask = TcpConnection.MultiDocumentParser.ParseToMemoryAsync("client reply");
                    using (RegisterForNotificationOnNewDocuments(criteria))
                    {
                        var patch = SetupFilterScript(criteria);

                        while (CancellationTokenSource.IsCancellationRequested == false)
                        {
                            dbContext.ResetAndRenew();

                            bool anyDocumentsSentInCurrentIteration = false;
                            using (dbContext.OpenReadTransaction())
                            {
                                var documents = TcpConnection.DocumentDatabase.DocumentsStorage.GetDocumentsFrom(dbContext,
                                                                                                                 criteria.Collection,
                                                                                                                 startEtag + 1, 0, _options.MaxDocsPerBatch);
                                _buffer.SetLength(0);
                                var docsToFlush = 0;

                                var sendingCurrentBatchStopwatch = Stopwatch.StartNew();
                                foreach (var doc in documents)
                                {
                                    anyDocumentsSentInCurrentIteration = true;
                                    startEtag = doc.Etag;
                                    if (DocumentMatchCriteriaScript(patch, dbContext, doc) == false)
                                    {
                                        // make sure that if we read a lot of irrelevant documents, we send keep alive over the network
                                        if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
                                        {
                                            await SendHeartBeat();

                                            sendingCurrentBatchStopwatch.Reset();
                                        }
                                        continue;
                                    }
                                    doc.EnsureMetadata();

                                    TcpConnection.Context.Write(_bufferedWriter, new DynamicJsonValue
                                    {
                                        ["Type"] = "Data",
                                        ["Data"] = doc.Data
                                    });
                                    docsToFlush++;

                                    // perform flush for current batch after 1000ms of running
                                    if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
                                    {
                                        if (docsToFlush > 0)
                                        {
                                            await FlushDocsToClient(docsToFlush);

                                            docsToFlush = 0;
                                            sendingCurrentBatchStopwatch.Reset();
                                        }
                                        else
                                        {
                                            await SendHeartBeat();
                                        }
                                    }
                                }

                                if (anyDocumentsSentInCurrentIteration)
                                {
                                    TcpConnection.Context.Write(_bufferedWriter, new DynamicJsonValue
                                    {
                                        ["Type"] = "EndOfBatch"
                                    });

                                    await FlushDocsToClient(docsToFlush, true);
                                }

                                if (anyDocumentsSentInCurrentIteration == false)
                                {
                                    if (await WaitForChangedDocuments(replyFromClientTask))
                                    {
                                        continue;
                                    }
                                }

                                SubscriptionConnectionClientMessage clientReply;

                                while (true)
                                {
                                    var result =
                                        await Task.WhenAny(replyFromClientTask, Task.Delay(TimeSpan.FromSeconds(5), CancellationTokenSource.Token));

                                    if (result == replyFromClientTask)
                                    {
                                        using (var reply = await replyFromClientTask)
                                        {
                                            TcpConnection.RegisterBytesReceived(reply.Size);
                                            clientReply = JsonDeserializationServer.SubscriptionConnectionClientMessage(reply);
                                        }

                                        TcpConnection.ResetAndRenew();

                                        replyFromClientTask = TcpConnection.MultiDocumentParser.ParseToMemoryAsync("client reply");
                                        break;
                                    }
                                    await SendHeartBeat();
                                }
                                switch (clientReply.Type)
                                {
                                case SubscriptionConnectionClientMessage.MessageType.Acknowledge:
                                    TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed(_options.SubscriptionId,
                                                                                                                 clientReply.Etag);
                                    Stats.LastAckReceivedAt = DateTime.UtcNow;
                                    Stats.AckRate.Mark();
                                    await WriteJsonAsync(new DynamicJsonValue
                                    {
                                        ["Type"] = "Confirm",
                                        ["Etag"] = clientReply.Etag
                                    });

                                    break;

                                default:
                                    throw new ArgumentException("Unknown message type from client " +
                                                                clientReply.Type);
                                }
                            }
                        }
                    }
                }
        }