public StreamSource(Stream stream, DocumentsOperationContext context, DocumentDatabase database) { _peepingTomStream = new PeepingTomStream(stream, context); _context = context; _database = database; _log = LoggingSource.Instance.GetLogger <StreamSource>(database.Name); }
public YieldStreamResults(InMemoryDocumentSessionOperations session, StreamResult stream, bool isQueryStream) { _response = stream; _peepingTomStream = new PeepingTomStream(_response.Stream, session.Context); _session = session; _isQueryStream = isQueryStream; }
public static async Task <BlittableJsonReaderObject> SimulateSavingToFileAndLoadingAsync(JsonOperationContext context, BlittableJsonReaderObject toStream) { //Simulates saving to file and loading BlittableJsonReaderObject fromStream; await using (Stream stream = new MemoryStream()) { //Pass to stream await using (var textWriter = new AsyncBlittableJsonTextWriter(context, stream)) { context.Write(textWriter, toStream); } //Get from stream stream.Position = 0; var state = new JsonParserState(); var parser = new UnmanagedJsonParser(context, state, "some tag"); var peepingTomStream = new PeepingTomStream(stream, context); using (context.GetMemoryBuffer(out var buffer)) using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "some tag", parser, state)) { await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer); await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer); fromStream = builder.CreateReader(); } } return(fromStream); }
public TimeSeriesStreamEnumerator(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, CancellationToken token = default) { _context = context; _peepingTomStream = peepingTomStream; _parser = parser; _state = state; _buffer = buffer; _token = token; }
public TimeSeriesStreamEnumerator(InMemoryDocumentSessionOperations session, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, CancellationToken token = default) { _session = session; _peepingTomStream = peepingTomStream; _parser = parser; _state = state; _buffer = buffer; _token = token; }
public void PeepingTomStreamTest(int originalSize, int chunkSizeToRead, int offset, JsonOperationContext context) { var bytes = new byte[originalSize]; for (var i = 0; i < bytes.Length; i++) { bytes[i] = (byte)((i % 26) + 'a'); } using (var stream = new MemoryStream()) using (var peeping = new PeepingTomStream(stream, context)) { stream.Write(bytes, 0, originalSize); stream.Flush(); stream.Position = 0; var totalRead = 0; do { int read; do { var buffer = new byte[originalSize + offset]; read = peeping.Read(buffer, offset, chunkSizeToRead); totalRead += read; Assert.True(read <= chunkSizeToRead); } while (read != 0); } while (totalRead < originalSize); Assert.Equal(originalSize, totalRead); var peepWindow = peeping.PeepInReadStream(); var length = peepWindow.Length; Assert.True(length <= PeepingTomStream.BufferWindowSize); Assert.True(length >= 0); var expectedLength = originalSize < PeepingTomStream.BufferWindowSize ? originalSize : PeepingTomStream.BufferWindowSize; if (expectedLength != length) { var expected = System.Text.Encoding.UTF8.GetString(bytes, bytes.Length - expectedLength, expectedLength); var actual = System.Text.Encoding.UTF8.GetString(peepWindow); Assert.Equal(expected, actual); } Assert.Equal(expectedLength, length); for (var i = 0; i < peepWindow.Length; i++) { var expectedByte = (byte)(((originalSize - peepWindow.Length + i) % 26) + 'a'); if (expectedByte != peepWindow[i]) { Assert.Equal(expectedByte, peepWindow[i]); } } } }
public IEnumerator <BlittableJsonReaderObject> SetResult(StreamResult response) { if (response == null) { throw new InvalidOperationException("The index does not exists, failed to stream results"); } var state = new JsonParserState(); JsonOperationContext.ManagedPinnedBuffer buffer; using (response.Response) using (response.Stream) using (var parser = new UnmanagedJsonParser(_session.Context, state, "stream contents")) using (_session.Context.GetManagedBuffer(out buffer)) using (var peepingTomStream = new PeepingTomStream(response.Stream, _session.Context)) { if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } if (state.CurrentTokenType != JsonParserToken.StartObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } if (_isQueryStream) { HandleStreamQueryStats(_session.Context, response, parser, state, buffer, _statistics); } var property = UnmanagedJsonParserHelper.ReadString(_session.Context, peepingTomStream, parser, state, buffer); if (string.Equals(property, "Results") == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } foreach (var result in UnmanagedJsonParserHelper.ReadArrayToMemory(_session.Context, peepingTomStream, parser, state, buffer)) { yield return(result); } if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } if (state.CurrentTokenType != JsonParserToken.EndObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } } }
public YieldStreamResults(InMemoryDocumentSessionOperations session, StreamResult response, bool isQueryStream, bool isTimeSeriesStream, bool isAsync, StreamQueryStatistics streamQueryStatistics, CancellationToken token = default) { _response = response ?? throw new InvalidOperationException("The index does not exists, failed to stream results"); _peepingTomStream = new PeepingTomStream(_response.Stream, session.Context); _session = session; _isQueryStream = isQueryStream; _isAsync = isAsync; _streamQueryStatistics = streamQueryStatistics; _maxDocsCountOnCachedRenewSession = session._maxDocsCountOnCachedRenewSession; _token = token; _isTimeSeriesStream = isTimeSeriesStream; }
public override void SetResponseRaw(HttpResponseMessage response, Stream stream, JsonOperationContext context) { var state = new JsonParserState(); using (var parser = new UnmanagedJsonParser(context, state, "multi_get/response")) using (context.GetMemoryBuffer(out var buffer)) using (var peepingTomStream = new PeepingTomStream(stream, context)) using (_cached) { if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false) { ThrowInvalidJsonResponse(peepingTomStream); } if (state.CurrentTokenType != JsonParserToken.StartObject) { ThrowInvalidJsonResponse(peepingTomStream); } var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (property != nameof(BlittableArrayResult.Results)) { ThrowInvalidJsonResponse(peepingTomStream); } var i = 0; Result = new List <GetResponse>(_commands.Count); foreach (var getResponse in ReadResponses(context, peepingTomStream, parser, state, buffer)) { var command = _commands[i]; MaybeSetCache(getResponse, command); Result.Add(_cached != null && getResponse.StatusCode == HttpStatusCode.NotModified ? new GetResponse { Result = _cached.Values[i].Cached.Clone(context), StatusCode = HttpStatusCode.NotModified } : getResponse); i++; } if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false) { ThrowInvalidJsonResponse(peepingTomStream); } if (state.CurrentTokenType != JsonParserToken.EndObject) { ThrowInvalidJsonResponse(peepingTomStream); } } }
public ReplayTransactionsException(string message, PeepingTomStream peepingTomStream) : base(message) { try { Context = Encodings.Utf8.GetString(peepingTomStream.PeepInReadStream()); } catch (Exception e) { Context = "Failed to generated peepedWindow: " + e; } }
public static unsafe string ReadString(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer) { if (Read(peepingTomStream, parser, state, buffer) == false) { ThrowInvalidJson(peepingTomStream); } if (state.CurrentTokenType != JsonParserToken.String) { ThrowInvalidJson(peepingTomStream); } return(context.AllocateStringValue(null, state.StringBuffer, state.StringSize).ToString()); }
private static string GetPeepingTomBufferAsString(PeepingTomStream peepingTomStream) { string peepedWindow; try { peepedWindow = Encodings.Utf8.GetString(peepingTomStream.PeepInReadStream()); } catch (Exception e) { peepedWindow = "Failed to generated peepedWindow: " + e; } return(peepedWindow); }
public static long ReadLong(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer) { if (Read(peepingTomStream, parser, state, buffer) == false) { ThrowInvalidJson(peepingTomStream); } if (state.CurrentTokenType != JsonParserToken.Integer) { ThrowInvalidJson(peepingTomStream); } return(state.Long); }
public YieldStreamResults(InMemoryDocumentSessionOperations session, StreamResult response, bool isQueryStream, bool isAsync, StreamQueryStatistics streamQueryStatistics) { if (response == null) { throw new InvalidOperationException("The index does not exists, failed to stream results"); } _response = response; _peepingTomStream = new PeepingTomStream(_response.Stream, session.Context); _session = session; _isQueryStream = isQueryStream; _isAsync = isAsync; _streamQueryStatistics = streamQueryStatistics; }
public override void SetResponseRaw(HttpResponseMessage response, Stream stream, JsonOperationContext context) { var state = new JsonParserState(); using (var parser = new UnmanagedJsonParser(context, state, "multi_get/response")) using (context.GetManagedBuffer(out JsonOperationContext.ManagedPinnedBuffer buffer)) using (var peepingTomStream = new PeepingTomStream(stream, context)) { if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false) { ThrowInvalidJsonResponse(peepingTomStream); } if (state.CurrentTokenType != JsonParserToken.StartObject) { ThrowInvalidJsonResponse(peepingTomStream); } var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (property != nameof(BlittableArrayResult.Results)) { ThrowInvalidJsonResponse(peepingTomStream); } var i = 0; Result = new List <GetResponse>(); foreach (var getResponse in ReadResponses(context, peepingTomStream, parser, state, buffer)) { var command = _commands[i]; MaybeSetCache(getResponse, command); MaybeReadFromCache(getResponse, command, context); Result.Add(getResponse); i++; } if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false) { ThrowInvalidJsonResponse(peepingTomStream); } if (state.CurrentTokenType != JsonParserToken.EndObject) { ThrowInvalidJsonResponse(peepingTomStream); } } }
private static void HandleStreamQueryStats(JsonOperationContext context, StreamResult response, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, StreamQueryStatistics streamQueryStatistics = null) { using (var peepingTomStream = new PeepingTomStream(response.Stream, context)) { var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.ResultEtag)) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); var resultEtag = UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer); property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.IsStale)) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); if (state.CurrentTokenType != JsonParserToken.False && state.CurrentTokenType != JsonParserToken.True) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); var isStale = state.CurrentTokenType != JsonParserToken.False; property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.IndexName)) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); var indexName = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.TotalResults)) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); var totalResults = (int)UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer); property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.IndexTimestamp)) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); var indexTimestamp = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (streamQueryStatistics == null) return; streamQueryStatistics.IndexName = indexName; streamQueryStatistics.IsStale = isStale; streamQueryStatistics.TotalResults = totalResults; streamQueryStatistics.ResultEtag = resultEtag; if (DateTime.TryParseExact(indexTimestamp, "o", CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out DateTime timeStamp) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); streamQueryStatistics.IndexTimestamp = timeStamp; } }
public static void ReadObject(BlittableJsonDocumentBuilder builder, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonOperationContext.ManagedPinnedBuffer buffer) { builder.ReadNestedObject(); while (builder.Read() == false) { var read = peepingTomStream.Read(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length); if (read == 0) { throw new EndOfStreamException("Stream ended without reaching end of json content" + GetPeepingTomBufferAsString(peepingTomStream)); } parser.SetBuffer(buffer, 0, read); } builder.FinalizeDocument(); }
public static void ReadProperty(BlittableJsonDocumentBuilder builder, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonOperationContext.MemoryBuffer buffer) { builder.ReadProperty(); while (builder.Read() == false) { var read = peepingTomStream.Read(buffer.Memory.Memory.Span); if (read == 0) { throw new EndOfStreamException("Stream ended without reaching end of json content" + GetPeepingTomBufferAsString(peepingTomStream)); } parser.SetBuffer(buffer, 0, read); } builder.FinalizeDocument(); }
public override async Task <ResponseDisposeHandling> ProcessResponse(JsonOperationContext context, HttpCache cache, HttpResponseMessage response, string url) { var state = new JsonParserState(); Stream stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); using (context.GetMemoryBuffer(out var buffer)) using (var parser = new UnmanagedJsonParser(context, state, "attachments/receive")) using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "attachments/list", parser, state)) using (var peepingTomStream = new PeepingTomStream(stream, context)) { if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false) { throw new Exception("cannot parse stream"); } if (state.CurrentTokenType != JsonParserToken.StartObject) { throw new Exception($"Expected token {nameof(JsonParserToken.StartObject)}, but got {nameof(state.CurrentTokenType)}."); } await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false); using (var data = builder.CreateReader()) { if (data.TryGetMember(nameof(AttachmentsMetadata), out object obj) && obj is BlittableJsonReaderArray bjra) { foreach (BlittableJsonReaderObject e in bjra) { var cur = JsonDeserializationClient.AttachmentDetails(e); AttachmentsMetadata.Add(cur); } } } var bufferSize = parser.BufferSize - parser.BufferOffset; var copy = ArrayPool <byte> .Shared.Rent(bufferSize); var copyMemory = new Memory <byte>(copy); buffer.Memory.Memory.Slice(parser.BufferOffset, bufferSize).CopyTo(copyMemory); Result = Iterate(stream, copy, bufferSize).GetEnumerator(); } return(ResponseDisposeHandling.Manually); }
public static bool Read(PeepingTomStream stream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer) { while (parser.Read() == false) { var read = stream.Read(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length); if (read == 0) { if (state.CurrentTokenType != JsonParserToken.EndObject) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } return(false); } parser.SetBuffer(buffer, 0, read); } return(true); }
public static async Task <bool> ReadAsync(PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, CancellationToken token = default) { while (parser.Read() == false) { var read = await peepingTomStream.ReadAsync(buffer.Memory.Memory, token).ConfigureAwait(false); if (read == 0) { if (state.CurrentTokenType != JsonParserToken.EndObject) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } return(false); } parser.SetBuffer(buffer, 0, read); } return(true); }
private static TransactionOperationsMerger.MergedTransactionCommand DeserializeCommand( DocumentsOperationContext context, DocumentDatabase database, string type, BlittableJsonReaderObject wrapCmdReader, PeepingTomStream peepingTomStream) { if (!wrapCmdReader.TryGet(nameof(RecordingCommandDetails.Command), out BlittableJsonReaderObject commandReader)) { throw new ReplayTransactionsException($"Can't read {type} for replay", peepingTomStream); } var jsonSerializer = GetJsonSerializer(); using (var reader = new BlittableJsonReader(context)) { reader.Init(commandReader); var dto = DeserializeCommandDto(type, jsonSerializer, reader, peepingTomStream); return(dto.ToCommand(context, database)); } }
public static async Task <bool> ReadAsync(PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer) { if (parser.Read()) { return(true); } var read = await peepingTomStream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length).ConfigureAwait(false); if (read == 0) { if (state.CurrentTokenType != JsonParserToken.EndObject) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } return(false); } parser.SetBuffer(buffer, 0, read); return(parser.Read()); }
private async Task ProcessChanges() { using (_requestExecutor.ContextPool.AllocateOperationContext(out var context)) { while (_cts.IsCancellationRequested == false) { context.Reset(); context.Renew(); var state = new JsonParserState(); using (var stream = new WebSocketStream(_client, _cts.Token)) using (context.GetMemoryBuffer(out JsonOperationContext.MemoryBuffer buffer)) using (var parser = new UnmanagedJsonParser(context, state, "changes/receive")) using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", parser, state)) using (var peepingTomStream = new PeepingTomStream(stream, context)) { if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false) { continue; } if (state.CurrentTokenType != JsonParserToken.StartArray) { continue; } while (true) { builder.Reset(); builder.Renew("changes/receive", BlittableJsonDocumentBuilder.UsageMode.None); if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false) { continue; } if (state.CurrentTokenType == JsonParserToken.EndArray) { break; } await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false); using (var json = builder.CreateReader()) { try { if (json.TryGet(nameof(TopologyChange), out bool supports) && supports) { GetOrAddConnectionState("Topology", "watch-topology-change", "", ""); await _requestExecutor.UpdateTopologyAsync(new RequestExecutor.UpdateTopologyParameters(_serverNode) { TimeoutInMs = 0, ForceUpdate = true, DebugTag = "watch-topology-change" }).ConfigureAwait(false); continue; } if (json.TryGet("Type", out string type) == false) { continue; } switch (type) { case "Error": json.TryGet("Exception", out string exceptionAsString); NotifyAboutError(new Exception(exceptionAsString)); break; case "Confirm": if (json.TryGet("CommandId", out int commandId) && _confirmations.TryRemove(commandId, out var tcs)) { tcs.TrySetResult(null); } break; default: json.TryGet("Value", out BlittableJsonReaderObject value); NotifySubscribers(type, value, _counters.ForceEnumerateInThreadSafeManner().Select(x => x.Value).ToList()); break; } } catch (Exception e) { NotifyAboutError(e); throw new ChangeProcessingException(e); } } } } } } }
public void JsonDeserialize_WhenHasBlittableObjectPropertyAndWriteAndReadFromStream_ShouldResultInCommandWithTheProperty() { using (Server.ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { var jsonSerializer = new JsonSerializer { ContractResolver = new DefaultRavenContractResolver(DocumentConventions.Default.Serialization), }; jsonSerializer.Converters.Add(BlittableJsonConverter.Instance); var data = new { SomeProperty = "SomeValue" }; var expected = DocumentConventions.Default.Serialization.DefaultConverter.ToBlittable(data, context); var command = new Command { BlittableObject = expected }; //Serialize BlittableJsonReaderObject toStream; using (var writer = new BlittableJsonWriter(context)) { jsonSerializer.Serialize(writer, command); writer.FinalizeDocument(); toStream = writer.CreateReader(); } //Simulates copying to file and loading BlittableJsonReaderObject fromStream; using (Stream stream = new MemoryStream()) { //Pass to stream using (var textWriter = new BlittableJsonTextWriter(context, stream)) { context.Write(textWriter, toStream); } //Get from stream stream.Position = 0; var state = new JsonParserState(); var parser = new UnmanagedJsonParser(context, state, "some tag"); var peepingTomStream = new PeepingTomStream(stream, context); using (context.GetMemoryBuffer(out var buffer)) using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "some tag", parser, state)) { UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer); UnmanagedJsonParserHelper.ReadObject(builder, peepingTomStream, parser, buffer); fromStream = builder.CreateReader(); } } //Deserialize BlittableJsonReaderObject actual; using (var reader = new BlittableJsonReader(context)) { reader.Initialize(fromStream); var deserialized = jsonSerializer.Deserialize <Command>(reader); actual = deserialized.BlittableObject; } Assert.Equal(expected, actual); } }
private async Task ProcessChanges() { using (_requestExecutor.ContextPool.AllocateOperationContext(out var context)) { while (_cts.IsCancellationRequested == false) { var state = new JsonParserState(); using (var stream = new WebSocketStream(_client, _cts.Token)) using (context.GetManagedBuffer(out var buffer)) using (var parser = new UnmanagedJsonParser(context, state, "changes/receive")) using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", parser, state)) using (var peepingTomStream = new PeepingTomStream(stream, context)) { if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false) { continue; } if (state.CurrentTokenType != JsonParserToken.StartArray) { continue; } while (true) { if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false) { continue; } if (state.CurrentTokenType == JsonParserToken.EndArray) { break; } builder.Renew("changes/receive", BlittableJsonDocumentBuilder.UsageMode.None); await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false); var json = builder.CreateReader(); try { if (json.TryGet("Type", out string type) == false) { continue; } switch (type) { case "Error": json.TryGet("Exception", out string exceptionAsString); NotifyAboutError(new Exception(exceptionAsString)); break; case "Confirm": if (json.TryGet("CommandId", out int commandId) && _confirmations.TryRemove(commandId, out var tcs)) { tcs.TrySetResult(null); } break; default: json.TryGet("Value", out BlittableJsonReaderObject value); NotifySubscribers(type, value, _counters.ValuesSnapshot); break; } } catch (Exception e) { NotifyAboutError(e); throw new ChangeProcessingException(e); } } } } } }
public async Task ConnectInternal() { try { var urlBuilder = new StringBuilder(_url).Append("/admin/traffic-watch"); if (string.IsNullOrWhiteSpace(_database) == false) { urlBuilder.Append("?resourceName=").Append(_database); } var stringUrl = ToWebSocketPath(urlBuilder.ToString().ToLower()); var url = new Uri(stringUrl, UriKind.Absolute); _client = new ClientWebSocket(); if (_cert != null) { _client.Options.ClientCertificates.Add(_cert); } await _client.ConnectAsync(url, _cancellationTokenSource.Token).ConfigureAwait(false); _firstConnection = false; Console.WriteLine($"Connected to RavenDB server. Collecting traffic watch entries to {_path}"); const int maxFileSize = 128 * 1024 * 1024; while (_cancellationTokenSource.IsCancellationRequested == false) { string file = Path.Combine(_path, _logNameCreator.GetNewFileName()); var state = new JsonParserState(); using (var context = JsonOperationContext.ShortTermSingleUse()) // Read await using (var stream = new WebSocketStream(_client, _cancellationTokenSource.Token)) using (context.GetMemoryBuffer(out var buffer)) using (var parser = new UnmanagedJsonParser(context, state, "trafficwatch/receive")) using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readObject/singleResult", parser, state)) // Write await using (var fileStream = new FileStream(file, FileMode.Append, FileAccess.Write, FileShare.Read, 32 * 1024, false)) await using (var gZipStream = new GZipStream(fileStream, CompressionMode.Compress, false)) using (var peepingTomStream = new PeepingTomStream(stream, context)) await using (var writer = new AsyncBlittableJsonTextWriter(context, gZipStream)) { writer.WriteStartArray(); var isFirst = true; while (fileStream.Length < maxFileSize) { if (_cancellationTokenSource.IsCancellationRequested) { writer.WriteEndArray(); break; } try { var flushCount = 0; while (fileStream.Length < maxFileSize && _cancellationTokenSource.IsCancellationRequested == false) { builder.Reset(); builder.Renew("trafficwatch/receive", BlittableJsonDocumentBuilder.UsageMode.None); if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false) { continue; } await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false); using (var json = builder.CreateReader()) { if (_changeTypes != null) { if (json.TryGet("Type", out TrafficWatchChangeType type) == false) { continue; } if (_changeTypes.Contains(type) == false) { continue; } } if (_database != null) { if (json.TryGet("DatabaseName", out LazyStringValue databaseName) == false || _database.Equals(databaseName, StringComparison.OrdinalIgnoreCase) == false) { continue; } } if (isFirst == false) { writer.WriteComma(); } isFirst = false; if (_verbose) { Console.WriteLine(json); } writer.WriteObject(json); _errorCount = 0; if (flushCount++ % 128 == 0) { await writer.FlushAsync(); } } } } catch (Exception) { writer.WriteEndArray(); throw; } } } } } catch (ObjectDisposedException) { // closing } }
private static void ReadStartRecordingDetails(IEnumerator <BlittableJsonReaderObject> iterator, DocumentsOperationContext context, PeepingTomStream peepingTomStream) { if (false == iterator.MoveNext()) { throw new ReplayTransactionsException("Replay stream is empty", peepingTomStream); } using (iterator.Current) { var jsonSerializer = GetJsonSerializer(); StartRecordingDetails startDetail; using (var reader = new BlittableJsonReader(context)) { reader.Init(iterator.Current); startDetail = jsonSerializer.Deserialize <StartRecordingDetails>(reader); } if (string.IsNullOrEmpty(startDetail.Type)) { throw new ReplayTransactionsException($"Can't read {nameof(RecordingDetails.Type)} of replay detail", peepingTomStream); } if (string.IsNullOrEmpty(startDetail.Type)) { throw new ReplayTransactionsException($"Can't read {nameof(StartRecordingDetails.Version)} of replay instructions", peepingTomStream); } if (startDetail.Version != RavenVersionAttribute.Instance.Build) { throw new ReplayTransactionsException($"Can't replay transaction instructions of different server version - Current version({ServerVersion.FullVersion}), Record version({startDetail.Version})", peepingTomStream); } } }
internal static IEnumerable <ReplayProgress> Replay(DocumentDatabase database, Stream replayStream) { DocumentsOperationContext txCtx = null; IDisposable txDisposable = null; DocumentsTransaction previousTx = null; using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.GetManagedBuffer(out var buffer)) using (var gZipStream = new GZipStream(replayStream, CompressionMode.Decompress, leaveOpen: true)) { var peepingTomStream = new PeepingTomStream(gZipStream, context); var state = new JsonParserState(); var parser = new UnmanagedJsonParser(context, state, "file"); var commandsProgress = 0; var readers = UnmanagedJsonParserHelper.ReadArrayToMemory(context, peepingTomStream, parser, state, buffer); using (var readersItr = readers.GetEnumerator()) { ReadStartRecordingDetails(readersItr, context, peepingTomStream); while (readersItr.MoveNext()) { using (readersItr.Current) { if (readersItr.Current.TryGet(nameof(RecordingDetails.Type), out string strType) == false) { throw new ReplayTransactionsException($"Can't read {nameof(RecordingDetails.Type)} of replay detail", peepingTomStream); } if (Enum.TryParse <TxInstruction>(strType, true, out var type)) { switch (type) { case TxInstruction.BeginTx: txDisposable = database.DocumentsStorage.ContextPool.AllocateOperationContext(out txCtx); txCtx.OpenWriteTransaction(); break; case TxInstruction.Commit: txCtx.Transaction.Commit(); break; case TxInstruction.DisposeTx: txDisposable.Dispose(); break; case TxInstruction.BeginAsyncCommitAndStartNewTransaction: previousTx = txCtx.Transaction; txCtx.Transaction = txCtx.Transaction.BeginAsyncCommitAndStartNewTransaction(txCtx); txDisposable = txCtx.Transaction; break; case TxInstruction.EndAsyncCommit: previousTx.EndAsyncCommit(); break; case TxInstruction.DisposePrevTx: previousTx.Dispose(); break; } continue; } try { var cmd = DeserializeCommand(context, database, strType, readersItr.Current, peepingTomStream); commandsProgress += cmd.ExecuteDirectly(txCtx); TransactionOperationsMerger.UpdateGlobalReplicationInfoBeforeCommit(txCtx); } catch (Exception) { //TODO To accept exceptions that was thrown while recording txDisposable.Dispose(); throw; } yield return(new ReplayProgress { CommandsProgress = commandsProgress }); } } } } }
private static TransactionOperationsMerger.IReplayableCommandDto <TransactionOperationsMerger.MergedTransactionCommand> DeserializeCommandDto( string type, JsonSerializer jsonSerializer, BlittableJsonReader reader, PeepingTomStream peepingTomStream) { switch (type) { case nameof(BatchHandler.MergedBatchCommand): return(jsonSerializer.Deserialize <MergedBatchCommandDto>(reader)); case nameof(DeleteDocumentCommand): return(jsonSerializer.Deserialize <DeleteDocumentCommandDto>(reader)); case nameof(PatchDocumentCommand): return(jsonSerializer.Deserialize <PatchDocumentCommandDto>(reader)); case nameof(DatabaseDestination.MergedBatchPutCommand): return(jsonSerializer.Deserialize <DatabaseDestination.MergedBatchPutCommandDto>(reader)); case nameof(MergedPutCommand): return(jsonSerializer.Deserialize <MergedPutCommand.MergedPutCommandDto>(reader)); case nameof(BulkInsertHandler.MergedInsertBulkCommand): return(jsonSerializer.Deserialize <MergedInsertBulkCommandDto>(reader)); case nameof(AttachmentHandler.MergedPutAttachmentCommand): return(jsonSerializer.Deserialize <MergedPutAttachmentCommandDto>(reader)); case nameof(AttachmentHandler.MergedDeleteAttachmentCommand): return(jsonSerializer.Deserialize <MergedDeleteAttachmentCommandDto>(reader)); case nameof(ResolveConflictOnReplicationConfigurationChange.PutResolvedConflictsCommand): return(jsonSerializer.Deserialize <PutResolvedConflictsCommandDto>(reader)); case nameof(HiLoHandler.MergedNextHiLoCommand): return(jsonSerializer.Deserialize <MergedNextHiLoCommandDto>(reader)); case nameof(HiLoHandler.MergedHiLoReturnCommand): return(jsonSerializer.Deserialize <MergedHiLoReturnCommandDto>(reader)); case nameof(IncomingReplicationHandler.MergedDocumentReplicationCommand): return(jsonSerializer.Deserialize <MergedDocumentReplicationCommandDto>(reader)); case nameof(ExpiredDocumentsCleaner.DeleteExpiredDocumentsCommand): return(jsonSerializer.Deserialize <DeleteExpiredDocumentsCommandDto>(reader)); case nameof(OutgoingReplicationHandler.UpdateSiblingCurrentEtag): return(jsonSerializer.Deserialize <UpdateSiblingCurrentEtagDto>(reader)); case nameof(IncomingReplicationHandler.MergedUpdateDatabaseChangeVectorCommand): return(jsonSerializer.Deserialize <MergedUpdateDatabaseChangeVectorCommandDto>(reader)); case nameof(AdminRevisionsHandler.DeleteRevisionsCommand): return(jsonSerializer.Deserialize <DeleteRevisionsCommandDto>(reader)); case nameof(RevisionsOperations.DeleteRevisionsBeforeCommand): throw new ReplayTransactionsException( "Because this command is deleting according to revisions' date & the revisions that created by replaying have different date an in place decision needed to be made", peepingTomStream); case nameof(TombstoneCleaner.DeleteTombstonesCommand): return(jsonSerializer.Deserialize <DeleteTombstonesCommandDto>(reader)); case nameof(OutputReduceIndexWriteOperation.OutputReduceToCollectionCommand): return(jsonSerializer.Deserialize <OutputReduceToCollectionCommandDto>(reader)); case nameof(BatchHandler.ClusterTransactionMergedCommand): return(jsonSerializer.Deserialize <ClusterTransactionMergedCommandDto>(reader)); case nameof(CountersHandler.ExecuteCounterBatchCommand): return(jsonSerializer.Deserialize <ExecuteCounterBatchCommandDto>(reader)); default: throw new ReplayTransactionsException($"Can't read {type} for replay", peepingTomStream); } }