示例#1
0
        private async Task <Stream> GetDebugInfoFromNodeAsync(
            JsonOperationContext context,
            RequestExecutor requestExecutor,
            IEnumerable <string> databaseNames)
        {
            var bodyJson = new DynamicJsonValue
            {
                [nameof(NodeDebugInfoRequestHeader.FromUrl)]       = ServerStore.GetNodeHttpServerUrl(),
                [nameof(NodeDebugInfoRequestHeader.DatabaseNames)] = databaseNames
            };

            await using (var ms = new MemoryStream())
                await using (var writer = new AsyncBlittableJsonTextWriter(context, ms))
                {
                    context.Write(writer, bodyJson);
                    await writer.FlushAsync();

                    await ms.FlushAsync();

                    var rawStreamCommand = new GetRawStreamResultCommand($"admin/debug/remote-cluster-info-package", ms);

                    await requestExecutor.ExecuteAsync(rawStreamCommand, context);

                    rawStreamCommand.Result.Position = 0;
                    return(rawStreamCommand.Result);
                }
        }
        public async Task <bool> SendStatsOrHeartbeatToWebSocket <TContext>(Task <WebSocketReceiveResult> receive, WebSocket webSocket, JsonContextPoolBase <TContext> contextPool, MemoryStream ms, int timeToWait) where TContext : JsonOperationContext
        {
            if (receive.IsCompleted || webSocket.State != WebSocketState.Open)
            {
                return(false);
            }

            var tuple = await Stats.TryDequeueAsync(TimeSpan.FromMilliseconds(timeToWait));

            if (tuple.Item1 == false)
            {
                await webSocket.SendAsync(WebSocketHelper.Heartbeat, WebSocketMessageType.Text, true, CancellationToken);

                return(true);
            }

            ms.SetLength(0);

            using (contextPool.AllocateOperationContext(out TContext context))
                await using (var writer = new AsyncBlittableJsonTextWriter(context, ms))
                {
                    WriteStats(tuple.Item2, writer, context);
                    await writer.FlushAsync(CancellationToken);
                }

            ms.TryGetBuffer(out ArraySegment <byte> bytes);
            await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken);

            return(true);
        }
示例#3
0
        private static async Task WriteForDatabase(ZipArchive archive, JsonOperationContext context, LocalEndpointClient localEndpointClient, string databaseName, string path = null)
        {
            var endpointParameters = new Dictionary <string, Microsoft.Extensions.Primitives.StringValues>
            {
                { "database", new Microsoft.Extensions.Primitives.StringValues(databaseName) }
            };

            foreach (var route in DebugInfoPackageUtils.Routes.Where(x => x.TypeOfRoute == RouteInformation.RouteType.Databases))
            {
                try
                {
                    var entry = archive.CreateEntry(DebugInfoPackageUtils.GetOutputPathFromRouteInformation(route, path ?? databaseName));
                    entry.ExternalAttributes = ((int)(FilePermissions.S_IRUSR | FilePermissions.S_IWUSR)) << 16;

                    await using (var entryStream = entry.Open())
                        await using (var writer = new AsyncBlittableJsonTextWriter(context, entryStream))
                        {
                            using (var endpointOutput = await localEndpointClient.InvokeAndReadObjectAsync(route, context, endpointParameters))
                            {
                                context.Write(writer, endpointOutput);
                                await writer.FlushAsync();

                                await entryStream.FlushAsync();
                            }
                        }
                }
                catch (Exception e)
                {
                    await DebugInfoPackageUtils.WriteExceptionAsZipEntryAsync(e, archive, path ?? databaseName);
                }
            }
        }
        private async Task <ArraySegment <byte> > ToByteArraySegmentAsync(TrafficWatchChange change)
        {
            var json = new DynamicJsonValue
            {
                [nameof(change.TimeStamp)]           = change.TimeStamp,
                [nameof(change.RequestId)]           = change.RequestId,
                [nameof(change.HttpMethod)]          = change.HttpMethod,
                [nameof(change.ElapsedMilliseconds)] = change.ElapsedMilliseconds,
                [nameof(change.ResponseStatusCode)]  = change.ResponseStatusCode,
                [nameof(change.RequestUri)]          = change.RequestUri,
                [nameof(change.AbsoluteUri)]         = change.AbsoluteUri,
                [nameof(change.DatabaseName)]        = change.DatabaseName,
                [nameof(change.CustomInfo)]          = change.CustomInfo,
                [nameof(change.Type)]     = change.Type,
                [nameof(change.ClientIP)] = change.ClientIP
            };

            _bufferStream.SetLength(0);
            await using (var writer = new AsyncBlittableJsonTextWriter(_context, _bufferStream))
            {
                _context.Write(writer, json);
                await writer.FlushAsync();

                _bufferStream.TryGetBuffer(out var bytes);
                return(bytes);
            }
        }
示例#5
0
        public async Task CanCompareLazyStringValueAndLazyCompressedStringValue()
        {
            using (var context = JsonOperationContext.ShortTermSingleUse())
                await using (var ms = new MemoryStream())
                    await using (var writer = new AsyncBlittableJsonTextWriter(context, ms))
                    {
                        writer.WriteStartObject();
                        writer.WritePropertyName("Test");
                        writer.WriteString(new string('c', 1024 * 1024));
                        writer.WriteEndObject();
                        await writer.FlushAsync();

                        await ms.FlushAsync();

                        ms.Position = 0;
                        var json = await context.ReadForDiskAsync(ms, "test");

                        ms.Position = 0;
                        var json2 = await context.ReadForDiskAsync(ms, "test");

                        var lcsv1 = (LazyCompressedStringValue)json["Test"];
                        var lcsv2 = (LazyCompressedStringValue)json2["Test"];
                        var lsv2  = lcsv2.ToLazyStringValue();

                        Assert.Equal(lcsv1, lsv2);
                        Assert.Equal(lsv2, lcsv1);
                        Assert.Equal(lsv2, lcsv2);
                        Assert.Equal(lcsv2, lsv2);
                    }
        }
示例#6
0
        private async Task WriteServerWide(ZipArchive archive, JsonOperationContext context, LocalEndpointClient localEndpointClient, string prefix)
        {
            //theoretically this could be parallelized,
            //however ZipArchive allows only one archive entry to be open concurrently
            foreach (var route in DebugInfoPackageUtils.Routes.Where(x => x.TypeOfRoute == RouteInformation.RouteType.None))
            {
                var entryRoute = DebugInfoPackageUtils.GetOutputPathFromRouteInformation(route, prefix);
                try
                {
                    var entry = archive.CreateEntry(entryRoute);
                    entry.ExternalAttributes = ((int)(FilePermissions.S_IRUSR | FilePermissions.S_IWUSR)) << 16;

                    await using (var entryStream = entry.Open())
                        await using (var writer = new AsyncBlittableJsonTextWriter(context, entryStream))
                            using (var endpointOutput = await localEndpointClient.InvokeAndReadObjectAsync(route, context))
                            {
                                context.Write(writer, endpointOutput);
                                await writer.FlushAsync();

                                await entryStream.FlushAsync();
                            }
                }
                catch (Exception e)
                {
                    await DebugInfoPackageUtils.WriteExceptionAsZipEntryAsync(e, archive, entryRoute);
                }
            }
        }
 private async ValueTask WriteClusterMaintenanceConnectionHeaderAsync(AsyncBlittableJsonTextWriter writer)
 {
     writer.WriteStartObject();
     {
         writer.WritePropertyName(nameof(ClusterMaintenanceConnectionHeader.LeaderClusterTag));
         writer.WriteString(_parent._leaderClusterTag);
         writer.WritePropertyName(nameof(ClusterMaintenanceConnectionHeader.Term));
         writer.WriteInteger(_parent._term);
     }
     writer.WriteEndObject();
     await writer.FlushAsync();
 }
示例#8
0
        public async Task GetInfoPackage()
        {
            var contentDisposition = $"attachment; filename={DateTime.UtcNow:yyyy-MM-dd H:mm:ss} - Database [{Database.Name}].zip";

            HttpContext.Response.Headers["Content-Disposition"] = contentDisposition;
            using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
            {
                await using (var ms = new MemoryStream())
                {
                    using (var archive = new ZipArchive(ms, ZipArchiveMode.Create, true))
                    {
                        var localEndpointClient = new LocalEndpointClient(Server);
                        var endpointParameters  = new Dictionary <string, Microsoft.Extensions.Primitives.StringValues>
                        {
                            { "database", new Microsoft.Extensions.Primitives.StringValues(Database.Name) }
                        };
                        var feature = HttpContext.Features.Get <IHttpAuthenticationFeature>() as RavenServer.AuthenticateConnection;
                        Debug.Assert(feature != null);
                        var routes = DebugInfoPackageUtils.GetAuthorizedRoutes(feature, Database.Name)
                                     .Where(x => x.TypeOfRoute == RouteInformation.RouteType.Databases);

                        foreach (RouteInformation route in routes)
                        {
                            var entryName = DebugInfoPackageUtils.GetOutputPathFromRouteInformation(route, null);
                            try
                            {
                                var entry = archive.CreateEntry(entryName);
                                entry.ExternalAttributes = ((int)(FilePermissions.S_IRUSR | FilePermissions.S_IWUSR)) << 16;

                                await using (var entryStream = entry.Open())
                                    await using (var writer = new AsyncBlittableJsonTextWriter(context, entryStream))
                                    {
                                        using (var endpointOutput = await localEndpointClient.InvokeAndReadObjectAsync(route, context, endpointParameters))
                                        {
                                            context.Write(writer, endpointOutput);
                                            await writer.FlushAsync();

                                            await entryStream.FlushAsync();
                                        }
                                    }
                            }
                            catch (Exception e)
                            {
                                await DebugInfoPackageUtils.WriteExceptionAsZipEntryAsync(e, archive, entryName.Replace(".json", string.Empty));
                            }
                        }
                    }

                    ms.Position = 0;
                    await ms.CopyToAsync(ResponseBodyStream());
                }
            }
        }
        private async Task <ArraySegment <byte> > ToByteArraySegmentAsync(TrafficWatchChangeBase change)
        {
            var json = change.ToJson();

            _bufferStream.SetLength(0);
            await using (var writer = new AsyncBlittableJsonTextWriter(_context, _bufferStream))
            {
                _context.Write(writer, json);
                await writer.FlushAsync();

                _bufferStream.TryGetBuffer(out var bytes);
                return(bytes);
            }
        }
示例#10
0
 private static async ValueTask WriteOperationHeaderToRemote(AsyncBlittableJsonTextWriter writer, TcpConnectionHeaderMessage.OperationTypes operation, string databaseName)
 {
     writer.WriteStartObject();
     {
         writer.WritePropertyName(nameof(TcpConnectionHeaderMessage.Operation));
         writer.WriteString(operation.ToString());
         writer.WriteComma();
         writer.WritePropertyName(nameof(TcpConnectionHeaderMessage.OperationVersion));
         writer.WriteInteger(TcpConnectionHeaderMessage.GetOperationTcpVersion(operation));
         writer.WriteComma();
         writer.WritePropertyName(nameof(TcpConnectionHeaderMessage.DatabaseName));
         writer.WriteString(databaseName);
     }
     writer.WriteEndObject();
     await writer.FlushAsync();
 }
        internal static async Task InvokeAndWriteToArchive(ZipArchive archive, JsonOperationContext jsonOperationContext, LocalEndpointClient localEndpointClient,
                                                           RouteInformation route, string path,
                                                           Dictionary <string, Microsoft.Extensions.Primitives.StringValues> endpointParameters = null,
                                                           CancellationToken token = default)
        {
            try
            {
                var response = await localEndpointClient.InvokeAsync(route, endpointParameters);

                var entryName = DebugInfoPackageUtils.GetOutputPathFromRouteInformation(route, path, response.ContentType == "text/plain" ? "txt" : "json");
                var entry     = archive.CreateEntry(entryName);
                entry.ExternalAttributes = ((int)(FilePermissions.S_IRUSR | FilePermissions.S_IWUSR)) << 16;

                using (var entryStream = entry.Open())
                {
                    if (response.ContentType == "text/plain")
                    {
                        await response.Body.CopyToAsync(entryStream, token);
                    }
                    else
                    {
                        await using (var writer = new AsyncBlittableJsonTextWriter(jsonOperationContext, entryStream))
                        {
                            var endpointOutput = await jsonOperationContext.ReadForMemoryAsync(response.Body, $"read/local endpoint/{route.Path}");

                            jsonOperationContext.Write(writer, endpointOutput);
                            await writer.FlushAsync();
                        }
                    }
                    await entryStream.FlushAsync(token);
                }
            }
            catch (OperationCanceledException)
            {
                throw;
            }
            catch (Exception e)
            {
                //precaution, ideally this exception should never be thrown
                if (e is InvalidStartOfObjectException)
                {
                    e = new InvalidOperationException("Expected to find a blittable object as a result of debug endpoint, but found something else (see inner exception for details). This should be investigated as all RavenDB endpoints are supposed to return an object.", e);
                }

                await DebugInfoPackageUtils.WriteExceptionAsZipEntryAsync(e, archive, DebugInfoPackageUtils.GetOutputPathFromRouteInformation(route, path, null));
            }
        }
示例#12
0
        public async Task ClusterMaintenanceStats()
        {
            if (ServerStore.LeaderTag == null)
            {
                return;
            }

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    if (ServerStore.IsLeader())
                    {
                        context.Write(writer, DynamicJsonValue.Convert(ServerStore.ClusterMaintenanceSupervisor?.GetStats()));
                        await writer.FlushAsync();

                        return;
                    }
                    RedirectToLeader();
                }
        }
示例#13
0
        public async Task ReadDocWithCompressedStringFromOneContextAndWriteToAnother()
        {
            using (var documentStore = GetDocumentStore())
            {
                Server.ServerStore.Observer.Suspended = true;
                var originalDoc = new Doc
                {
                    Id            = "doc/1",
                    StrVal        = new string(Enumerable.Repeat('.', 129).ToArray()),
                    LongByteArray = Enumerable.Repeat((byte)2, 1024).ToArray()
                };

                using (var session = documentStore.OpenAsyncSession())
                {
                    await session.StoreAsync(originalDoc);

                    await session.SaveChangesAsync();
                }

                var database = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(documentStore.Database);

                using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                    using (context.OpenReadTransaction())
                    {
                        var          doc = database.DocumentsStorage.Get(context, "doc/1");
                        MemoryStream ms  = new MemoryStream();
                        using (var newContext = JsonOperationContext.ShortTermSingleUse())
                            await using (var writer = new AsyncBlittableJsonTextWriter(newContext, ms))
                            {
                                writer.WriteDocument(newContext, doc, metadataOnly: false);
                                await writer.FlushAsync();

                                var bjro            = GetReaderFromMemoryStream(ms, context);
                                var deserializedDoc = (Doc)DocumentConventions.Default.Serialization.DefaultConverter.FromBlittable(typeof(Doc), bjro);

                                Assert.Equal(originalDoc.StrVal, deserializedDoc.StrVal);
                                Assert.Equal(originalDoc.LongByteArray, originalDoc.LongByteArray);
                            }
                    }
            }
        }
        public static async Task TrySavingAsync(string topologyHash, ClusterTopologyResponse clusterTopology, DocumentConventions conventions, JsonOperationContext context, CancellationToken token)
        {
            try
            {
                if (conventions.DisableTopologyCache)
                {
                    return;
                }

                var path = GetPath(topologyHash, conventions);
                if (clusterTopology == null)
                {
                    Clear(path);
                    return;
                }

                using (var stream = SafeFileStream.Create(path, FileMode.Create, FileAccess.Write, FileShare.Read))
                    await using (var writer = new AsyncBlittableJsonTextWriter(context, stream))
                    {
                        var json = new DynamicJsonValue
                        {
                            [nameof(clusterTopology.Topology)] = clusterTopology.Topology.ToJson(),
                            [nameof(clusterTopology.Leader)]   = clusterTopology.Leader,
                            [nameof(clusterTopology.NodeTag)]  = clusterTopology.NodeTag,
                            [nameof(clusterTopology.Etag)]     = clusterTopology.Etag,
                            ["PersistedAt"] = DateTimeOffset.UtcNow.ToString(DefaultFormat.DateTimeOffsetFormatsToWrite),
                        };

                        context.Write(writer, json);
                        await writer.FlushAsync(token).ConfigureAwait(false);
                    }
            }
            catch (Exception e)
            {
                if (_logger.IsInfoEnabled)
                {
                    _logger.Info("Could not persist the cluster topology", e);
                }
            }
        }
        private async Task WriteDatabaseRecord(ZipArchive archive, string databaseName, JsonOperationContext jsonOperationContext, TransactionOperationContext transactionCtx, CancellationToken token = default)
        {
            var entryName = DebugInfoPackageUtils.GetOutputPathFromRouteInformation("/database-record", databaseName, "json");

            try
            {
                var entry = archive.CreateEntry(entryName);
                entry.ExternalAttributes = ((int)(FilePermissions.S_IRUSR | FilePermissions.S_IWUSR)) << 16;

                await using (var entryStream = entry.Open())
                    await using (var writer = new AsyncBlittableJsonTextWriter(jsonOperationContext, entryStream))
                    {
                        jsonOperationContext.Write(writer, GetDatabaseRecordForDebugPackage(transactionCtx, databaseName));
                        await writer.FlushAsync();

                        await entryStream.FlushAsync(token);
                    }
            }
            catch (Exception e)
            {
                await DebugInfoPackageUtils.WriteExceptionAsZipEntryAsync(e, archive, entryName);
            }
        }
            private static async ValueTask WriteOperationHeaderToRemoteAsync(AsyncBlittableJsonTextWriter writer, int remoteVersion = -1, bool drop = false)
            {
                var operation = drop ? TcpConnectionHeaderMessage.OperationTypes.Drop : TcpConnectionHeaderMessage.OperationTypes.Heartbeats;

                writer.WriteStartObject();
                {
                    writer.WritePropertyName(nameof(TcpConnectionHeaderMessage.Operation));
                    writer.WriteString(operation.ToString());
                    writer.WriteComma();
                    writer.WritePropertyName(nameof(TcpConnectionHeaderMessage.OperationVersion));
                    writer.WriteInteger(TcpConnectionHeaderMessage.HeartbeatsTcpVersion);
                    writer.WriteComma();
                    writer.WritePropertyName(nameof(TcpConnectionHeaderMessage.DatabaseName));
                    writer.WriteString((string)null);
                    if (drop)
                    {
                        writer.WriteComma();
                        writer.WritePropertyName(nameof(TcpConnectionHeaderMessage.Info));
                        writer.WriteString($"Couldn't agree on heartbeats tcp version ours:{TcpConnectionHeaderMessage.HeartbeatsTcpVersion} theirs:{remoteVersion}");
                    }
                }
                writer.WriteEndObject();
                await writer.FlushAsync();
            }
示例#17
0
        public async Task StartSendingNotifications(bool throttleConnection)
        {
            using (_documentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out JsonOperationContext context))
            {
                await using (var ms = new MemoryStream())
                {
                    var sp = Stopwatch.StartNew();
                    while (true)
                    {
                        if (_disposeToken.IsCancellationRequested)
                        {
                            break;
                        }

                        ms.SetLength(0);
                        context.Reset();
                        context.Renew();

                        await using (var writer = new AsyncBlittableJsonTextWriter(context, ms))
                        {
                            sp.Restart();

                            var first = true;
                            writer.WriteStartArray();

                            do
                            {
                                var value = await GetNextMessage(throttleConnection);

                                if (value == null || _disposeToken.IsCancellationRequested)
                                {
                                    break;
                                }

                                if (first == false)
                                {
                                    writer.WriteComma();
                                }

                                first = false;
                                context.Write(writer, value);

                                await writer.FlushAsync();

                                if (ms.Length > 16 * 1024)
                                {
                                    break;
                                }
                            } while (_sendQueue.Count > 0 && sp.Elapsed < TimeSpan.FromSeconds(5));

                            writer.WriteEndArray();
                        }

                        if (_disposeToken.IsCancellationRequested)
                        {
                            break;
                        }

                        ms.TryGetBuffer(out ArraySegment <byte> bytes);
                        await _webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, _disposeToken);
                    }
                }
            }
        }
示例#18
0
        private async ValueTask HandleRequestAsync(
            BlittableJsonReaderObject request,
            JsonOperationContext context,
            MultiGetHttpResponseStream responseStream,
            AsyncBlittableJsonTextWriter writer,
            HttpContext httpContext,
            HostString host,
            string scheme,
            LazyStringValue resultProperty,
            LazyStringValue statusProperty,
            LazyStringValue headersProperty,
            StringBuilder trafficWatchStringBuilder)
        {
            writer.WriteStartObject();

            if (request.TryGet(nameof(GetRequest.Url), out string url) == false || request.TryGet(nameof(GetRequest.Query), out string query) == false)
            {
                writer.WriteEndObject();
                return;
            }

            if (request.TryGet(nameof(GetRequest.Method), out string method) == false || string.IsNullOrEmpty(method))
            {
                method = HttpMethod.Get.Method;
            }

            httpContext.Request.Method = method;

            var routeInformation = Server.Router.GetRoute(method, url, out RouteMatch localMatch);

            if (routeInformation == null)
            {
                HandleNoRoute(context, writer, method, url, query, statusProperty, resultProperty);
                return;
            }

            var requestHandler = routeInformation.GetRequestHandler();

            writer.WritePropertyName(resultProperty);
            await writer.FlushAsync();

            var content = await PrepareHttpContextAsync(request, context, httpContext, method, query, host, scheme, trafficWatchStringBuilder);

            var bytesWrittenBeforeRequest = responseStream.BytesWritten;
            int statusCode;

            try
            {
                if (Server.Configuration.Security.AuthenticationEnabled == false ||
                    (await Server.Router.TryAuthorizeAsync(routeInformation, httpContext, Database)).Authorized)
                {
                    await requestHandler(new RequestHandlerContext
                    {
                        Database    = Database,
                        RavenServer = Server,
                        RouteMatch  = localMatch,
                        HttpContext = httpContext
                    });
                }

                if (bytesWrittenBeforeRequest == responseStream.BytesWritten)
                {
                    writer.WriteNull();
                }

                statusCode = httpContext.Response.StatusCode == 0
                    ? (int)HttpStatusCode.OK
                    : httpContext.Response.StatusCode;
            }
            catch (Exception e)
            {
                if (bytesWrittenBeforeRequest != responseStream.BytesWritten)
                {
                    throw;
                }

                statusCode = (int)HttpStatusCode.InternalServerError;

                HandleException(context, writer, e, url, query);
            }

            writer.WriteComma();

            writer.WritePropertyName(statusProperty);
            writer.WriteInteger(statusCode);

            writer.WriteComma();

            WriteHeaders(writer, httpContext, headersProperty);

            writer.WriteEndObject();

            trafficWatchStringBuilder?.Append(content).AppendLine();
        }
示例#19
0
        public async Task ConnectInternal()
        {
            try
            {
                var urlBuilder = new StringBuilder(_url).Append("/admin/traffic-watch");

                if (string.IsNullOrWhiteSpace(_database) == false)
                {
                    urlBuilder.Append("?resourceName=").Append(_database);
                }

                var stringUrl = ToWebSocketPath(urlBuilder.ToString().ToLower());
                var url       = new Uri(stringUrl, UriKind.Absolute);
                _client = new ClientWebSocket();
                if (_cert != null)
                {
                    _client.Options.ClientCertificates.Add(_cert);
                }

                await _client.ConnectAsync(url, _cancellationTokenSource.Token).ConfigureAwait(false);

                _firstConnection = false;

                Console.WriteLine($"Connected to RavenDB server. Collecting traffic watch entries to {_path}");

                const int maxFileSize = 128 * 1024 * 1024;
                while (_cancellationTokenSource.IsCancellationRequested == false)
                {
                    string file = Path.Combine(_path, _logNameCreator.GetNewFileName());

                    var state = new JsonParserState();

                    using (var context = JsonOperationContext.ShortTermSingleUse())
                        // Read
                        await using (var stream = new WebSocketStream(_client, _cancellationTokenSource.Token))
                            using (context.GetMemoryBuffer(out var buffer))
                                using (var parser = new UnmanagedJsonParser(context, state, "trafficwatch/receive"))
                                    using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readObject/singleResult", parser, state))
                                        // Write
                                        await using (var fileStream = new FileStream(file, FileMode.Append, FileAccess.Write, FileShare.Read, 32 * 1024, false))
                                            await using (var gZipStream = new GZipStream(fileStream, CompressionMode.Compress, false))
                                                using (var peepingTomStream = new PeepingTomStream(stream, context))
                                                    await using (var writer = new AsyncBlittableJsonTextWriter(context, gZipStream))
                                                    {
                                                        writer.WriteStartArray();
                                                        var isFirst = true;

                                                        while (fileStream.Length < maxFileSize)
                                                        {
                                                            if (_cancellationTokenSource.IsCancellationRequested)
                                                            {
                                                                writer.WriteEndArray();
                                                                break;
                                                            }

                                                            try
                                                            {
                                                                var flushCount = 0;
                                                                while (fileStream.Length < maxFileSize && _cancellationTokenSource.IsCancellationRequested == false)
                                                                {
                                                                    builder.Reset();
                                                                    builder.Renew("trafficwatch/receive", BlittableJsonDocumentBuilder.UsageMode.None);

                                                                    if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
                                                                    {
                                                                        continue;
                                                                    }

                                                                    await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);

                                                                    using (var json = builder.CreateReader())
                                                                    {
                                                                        if (_changeTypes != null)
                                                                        {
                                                                            if (json.TryGet("Type", out TrafficWatchChangeType type) == false)
                                                                            {
                                                                                continue;
                                                                            }
                                                                            if (_changeTypes.Contains(type) == false)
                                                                            {
                                                                                continue;
                                                                            }
                                                                        }

                                                                        if (_database != null)
                                                                        {
                                                                            if (json.TryGet("DatabaseName", out LazyStringValue databaseName) == false ||
                                                                                _database.Equals(databaseName, StringComparison.OrdinalIgnoreCase) == false)
                                                                            {
                                                                                continue;
                                                                            }
                                                                        }

                                                                        if (isFirst == false)
                                                                        {
                                                                            writer.WriteComma();
                                                                        }

                                                                        isFirst = false;
                                                                        if (_verbose)
                                                                        {
                                                                            Console.WriteLine(json);
                                                                        }

                                                                        writer.WriteObject(json);
                                                                        _errorCount = 0;
                                                                        if (flushCount++ % 128 == 0)
                                                                        {
                                                                            await writer.FlushAsync();
                                                                        }
                                                                    }
                                                                }
                                                            }
                                                            catch (Exception)
                                                            {
                                                                writer.WriteEndArray();
                                                                throw;
                                                            }
                                                        }
                                                    }
                }
            }
            catch (ObjectDisposedException)
            {
                // closing
            }
        }
示例#20
0
        public async Task SubscriptionShouldRespectDocumentsWithCompressedData()
        {
            using (var documentStore = GetDocumentStore())
            {
                Server.ServerStore.Observer.Suspended = true;
                var originalDoc = new Doc
                {
                    Id            = "doc/1",
                    StrVal        = new string(Enumerable.Repeat('.', 129).ToArray()),
                    LongByteArray = Enumerable.Repeat((byte)2, 1024).ToArray()
                };

                using (var session = documentStore.OpenAsyncSession())
                {
                    await session.StoreAsync(originalDoc);

                    await session.SaveChangesAsync();
                }

                var database = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(documentStore.Database);

                using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                    using (context.OpenReadTransaction())
                    {
                        var          doc = database.DocumentsStorage.Get(context, "doc/1");
                        MemoryStream ms  = new MemoryStream();
                        using (var newContext = JsonOperationContext.ShortTermSingleUse())
                            await using (var writer = new AsyncBlittableJsonTextWriter(newContext, ms))
                            {
                                writer.WriteDocument(newContext, doc, metadataOnly: false);
                                await writer.FlushAsync();

                                var bjro            = GetReaderFromMemoryStream(ms, context);
                                var deserializedDoc = (Doc)DocumentConventions.Default.Serialization.DefaultConverter.FromBlittable(typeof(Doc), bjro);

                                Assert.Equal(originalDoc.StrVal, deserializedDoc.StrVal);
                                Assert.Equal(originalDoc.LongByteArray, originalDoc.LongByteArray);
                            }
                    }

                var subscriptionCreationParams = new SubscriptionCreationOptions
                {
                    Query = "from Docs",
                };

                var subsId = await documentStore.Subscriptions.CreateAsync(subscriptionCreationParams).ConfigureAwait(false);

                var amre = new AsyncManualResetEvent();
                await using (var subscription = documentStore.Subscriptions.GetSubscriptionWorker <Doc>(new SubscriptionWorkerOptions(subsId)
                {
                    TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
                }))
                {
                    var t = subscription.Run(batch =>
                    {
                        var receivedDoc = batch.Items.First().Result;
                        Assert.Equal(originalDoc.LongByteArray, receivedDoc.LongByteArray);
                        Assert.Equal(originalDoc.StrVal, receivedDoc.StrVal);
                        amre.Set();
                    });

                    try
                    {
                        Assert.True(await amre.WaitAsync(TimeSpan.FromSeconds(60)));
                    }
                    catch
                    {
                        if (t.IsFaulted)
                        {
                            t.Wait();
                        }
                        throw;
                    }
                }
            }
        }