public async Task <bool> MoveNextAsync() { if (_initialized == false) { await InitializeAsync().ConfigureAwait(false); } if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType != JsonParserToken.EndObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } return(false); } using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state)) { await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer).ConfigureAwait(false); Current = builder.CreateReader(); return(true); } }
public async ValueTask DisposeAsync() { if (_disposed) { return; } if (_done == false) { while (await MoveNextAsync().ConfigureAwait(false)) { // we need to consume the rest of the stream, before we can move next the outer enumerator } } if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType != JsonParserToken.EndObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } _disposed = true; }
public static async Task <BlittableJsonReaderObject> SimulateSavingToFileAndLoadingAsync(JsonOperationContext context, BlittableJsonReaderObject toStream) { //Simulates saving to file and loading BlittableJsonReaderObject fromStream; await using (Stream stream = new MemoryStream()) { //Pass to stream await using (var textWriter = new AsyncBlittableJsonTextWriter(context, stream)) { context.Write(textWriter, toStream); } //Get from stream stream.Position = 0; var state = new JsonParserState(); var parser = new UnmanagedJsonParser(context, state, "some tag"); var peepingTomStream = new PeepingTomStream(stream, context); using (context.GetMemoryBuffer(out var buffer)) using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "some tag", parser, state)) { await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer); await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer); fromStream = builder.CreateReader(); } } return(fromStream); }
public async Task <bool> MoveNextAsync() { if (_initialized == false) { await InitializeAsync().ConfigureAwait(false); } if (_docsCountOnCachedRenewSession <= 16 * 1024) { if (_cachedItemsRenew) { _session.Context.CachedProperties = new CachedProperties(_session.Context); ++_docsCountOnCachedRenewSession; } } else { _session.Context.Renew(); _docsCountOnCachedRenewSession = 0; } if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType != JsonParserToken.EndObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } return(false); } using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state)) { if (_cachedItemsRenew == false) { _cachedItemsRenew = builder.NeedResetPropertiesCache(); } await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer).ConfigureAwait(false); Current = builder.CreateReader(); return(true); } }
public async Task InitializeAsync() { var property = UnmanagedJsonParserHelper.ReadString(_context, _peepingTomStream, _parser, _state, _buffer); if (property.StartsWith(Constants.TimeSeries.QueryFunction) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType != JsonParserToken.StartArray) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); }
public async Task InitializeAsync() { AssertNotSync(); try { _initialized = true; _state = new JsonParserState(); _parser = new UnmanagedJsonParser(_session.Context, _state, "stream contents"); _returnBuffer = _session.Context.GetManagedBuffer(out _buffer); if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType != JsonParserToken.StartObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_isQueryStream) { HandleStreamQueryStats(_session.Context, _response, _parser, _state, _buffer, _streamQueryStatistics); } var property = UnmanagedJsonParserHelper.ReadString(_session.Context, _peepingTomStream, _parser, _state, _buffer); if (string.Equals(property, "Results") == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } } catch { Dispose(); throw; } }
public override async Task <ResponseDisposeHandling> ProcessResponse(JsonOperationContext context, HttpCache cache, HttpResponseMessage response, string url) { var state = new JsonParserState(); Stream stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); using (context.GetMemoryBuffer(out var buffer)) using (var parser = new UnmanagedJsonParser(context, state, "attachments/receive")) using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "attachments/list", parser, state)) using (var peepingTomStream = new PeepingTomStream(stream, context)) { if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false) { throw new Exception("cannot parse stream"); } if (state.CurrentTokenType != JsonParserToken.StartObject) { throw new Exception($"Expected token {nameof(JsonParserToken.StartObject)}, but got {nameof(state.CurrentTokenType)}."); } await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false); using (var data = builder.CreateReader()) { if (data.TryGetMember(nameof(AttachmentsMetadata), out object obj) && obj is BlittableJsonReaderArray bjra) { foreach (BlittableJsonReaderObject e in bjra) { var cur = JsonDeserializationClient.AttachmentDetails(e); AttachmentsMetadata.Add(cur); } } } var bufferSize = parser.BufferSize - parser.BufferOffset; var copy = ArrayPool <byte> .Shared.Rent(bufferSize); var copyMemory = new Memory <byte>(copy); buffer.Memory.Memory.Slice(parser.BufferOffset, bufferSize).CopyTo(copyMemory); Result = Iterate(stream, copy, bufferSize).GetEnumerator(); } return(ResponseDisposeHandling.Manually); }
public async ValueTask<bool> MoveNextAsync() { if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType == JsonParserToken.EndArray) { _done = true; _current = null; return false; } using (var builder = new BlittableJsonDocumentBuilder(_context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", _parser, _state)) { await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false); _current = builder.CreateReader(); return true; } }
public async ValueTask<bool> MoveNextAsync() { AssertInitialized(); CheckIfContextOrCacheNeedToBeRenewed(); if (_timeSeriesIt != null) await _timeSeriesIt.DisposeAsync().ConfigureAwait(false); if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType == JsonParserToken.EndArray) { if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType != JsonParserToken.EndObject) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); return false; } _builder.Renew("readArray/singleResult", BlittableJsonDocumentBuilder.UsageMode.ToDisk); if (_isTimeSeriesStream) await UnmanagedJsonParserHelper.ReadPropertyAsync(_builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false); else await UnmanagedJsonParserHelper.ReadObjectAsync(_builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false); Current = _builder.CreateReader(); _builder.Reset(); if (_isTimeSeriesStream) { _timeSeriesIt = new TimeSeriesStreamEnumerator(_builderContext, _peepingTomStream, _parser, _state, _buffer); await _timeSeriesIt.InitializeAsync().ConfigureAwait(false); } return true; }
public async Task InitializeAsync() { AssertNotSync(); try { _initialized = true; _state = new JsonParserState(); _parser = new UnmanagedJsonParser(_session.Context, _state, "stream contents"); _builder = new BlittableJsonDocumentBuilder(_builderContext, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state); _returnBuffer = _session.Context.GetMemoryBuffer(out _buffer); if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType != JsonParserToken.StartObject) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_isQueryStream) HandleStreamQueryStats(_builderContext, _response, _parser, _state, _buffer, _streamQueryStatistics); var property = UnmanagedJsonParserHelper.ReadString(_builderContext, _peepingTomStream, _parser, _state, _buffer); if (string.Equals(property, "Results") == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType != JsonParserToken.StartArray) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } catch { await DisposeAsync().ConfigureAwait(false); throw; } }
private async Task ProcessChanges() { using (_requestExecutor.ContextPool.AllocateOperationContext(out var context)) { while (_cts.IsCancellationRequested == false) { context.Reset(); context.Renew(); var state = new JsonParserState(); using (var stream = new WebSocketStream(_client, _cts.Token)) using (context.GetMemoryBuffer(out JsonOperationContext.MemoryBuffer buffer)) using (var parser = new UnmanagedJsonParser(context, state, "changes/receive")) using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", parser, state)) using (var peepingTomStream = new PeepingTomStream(stream, context)) { if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false) { continue; } if (state.CurrentTokenType != JsonParserToken.StartArray) { continue; } while (true) { builder.Reset(); builder.Renew("changes/receive", BlittableJsonDocumentBuilder.UsageMode.None); if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false) { continue; } if (state.CurrentTokenType == JsonParserToken.EndArray) { break; } await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false); using (var json = builder.CreateReader()) { try { if (json.TryGet(nameof(TopologyChange), out bool supports) && supports) { GetOrAddConnectionState("Topology", "watch-topology-change", "", ""); await _requestExecutor.UpdateTopologyAsync(new RequestExecutor.UpdateTopologyParameters(_serverNode) { TimeoutInMs = 0, ForceUpdate = true, DebugTag = "watch-topology-change" }).ConfigureAwait(false); continue; } if (json.TryGet("Type", out string type) == false) { continue; } switch (type) { case "Error": json.TryGet("Exception", out string exceptionAsString); NotifyAboutError(new Exception(exceptionAsString)); break; case "Confirm": if (json.TryGet("CommandId", out int commandId) && _confirmations.TryRemove(commandId, out var tcs)) { tcs.TrySetResult(null); } break; default: json.TryGet("Value", out BlittableJsonReaderObject value); NotifySubscribers(type, value, _counters.ForceEnumerateInThreadSafeManner().Select(x => x.Value).ToList()); break; } } catch (Exception e) { NotifyAboutError(e); throw new ChangeProcessingException(e); } } } } } } }
private async Task ProcessChanges() { using (_requestExecutor.ContextPool.AllocateOperationContext(out var context)) { while (_cts.IsCancellationRequested == false) { var state = new JsonParserState(); using (var stream = new WebSocketStream(_client, _cts.Token)) using (context.GetManagedBuffer(out var buffer)) using (var parser = new UnmanagedJsonParser(context, state, "changes/receive")) using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", parser, state)) using (var peepingTomStream = new PeepingTomStream(stream, context)) { if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false) { continue; } if (state.CurrentTokenType != JsonParserToken.StartArray) { continue; } while (true) { if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false) { continue; } if (state.CurrentTokenType == JsonParserToken.EndArray) { break; } builder.Renew("changes/receive", BlittableJsonDocumentBuilder.UsageMode.None); await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false); var json = builder.CreateReader(); try { if (json.TryGet("Type", out string type) == false) { continue; } switch (type) { case "Error": json.TryGet("Exception", out string exceptionAsString); NotifyAboutError(new Exception(exceptionAsString)); break; case "Confirm": if (json.TryGet("CommandId", out int commandId) && _confirmations.TryRemove(commandId, out var tcs)) { tcs.TrySetResult(null); } break; default: json.TryGet("Value", out BlittableJsonReaderObject value); NotifySubscribers(type, value, _counters.ValuesSnapshot); break; } } catch (Exception e) { NotifyAboutError(e); throw new ChangeProcessingException(e); } } } } } }
public async Task ConnectInternal() { try { var urlBuilder = new StringBuilder(_url).Append("/admin/traffic-watch"); if (string.IsNullOrWhiteSpace(_database) == false) { urlBuilder.Append("?resourceName=").Append(_database); } var stringUrl = ToWebSocketPath(urlBuilder.ToString().ToLower()); var url = new Uri(stringUrl, UriKind.Absolute); _client = new ClientWebSocket(); if (_cert != null) { _client.Options.ClientCertificates.Add(_cert); } await _client.ConnectAsync(url, _cancellationTokenSource.Token).ConfigureAwait(false); _firstConnection = false; Console.WriteLine($"Connected to RavenDB server. Collecting traffic watch entries to {_path}"); const int maxFileSize = 128 * 1024 * 1024; while (_cancellationTokenSource.IsCancellationRequested == false) { string file = Path.Combine(_path, _logNameCreator.GetNewFileName()); var state = new JsonParserState(); using (var context = JsonOperationContext.ShortTermSingleUse()) // Read await using (var stream = new WebSocketStream(_client, _cancellationTokenSource.Token)) using (context.GetMemoryBuffer(out var buffer)) using (var parser = new UnmanagedJsonParser(context, state, "trafficwatch/receive")) using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readObject/singleResult", parser, state)) // Write await using (var fileStream = new FileStream(file, FileMode.Append, FileAccess.Write, FileShare.Read, 32 * 1024, false)) await using (var gZipStream = new GZipStream(fileStream, CompressionMode.Compress, false)) using (var peepingTomStream = new PeepingTomStream(stream, context)) await using (var writer = new AsyncBlittableJsonTextWriter(context, gZipStream)) { writer.WriteStartArray(); var isFirst = true; while (fileStream.Length < maxFileSize) { if (_cancellationTokenSource.IsCancellationRequested) { writer.WriteEndArray(); break; } try { var flushCount = 0; while (fileStream.Length < maxFileSize && _cancellationTokenSource.IsCancellationRequested == false) { builder.Reset(); builder.Renew("trafficwatch/receive", BlittableJsonDocumentBuilder.UsageMode.None); if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false) { continue; } await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false); using (var json = builder.CreateReader()) { if (_changeTypes != null) { if (json.TryGet("Type", out TrafficWatchChangeType type) == false) { continue; } if (_changeTypes.Contains(type) == false) { continue; } } if (_database != null) { if (json.TryGet("DatabaseName", out LazyStringValue databaseName) == false || _database.Equals(databaseName, StringComparison.OrdinalIgnoreCase) == false) { continue; } } if (isFirst == false) { writer.WriteComma(); } isFirst = false; if (_verbose) { Console.WriteLine(json); } writer.WriteObject(json); _errorCount = 0; if (flushCount++ % 128 == 0) { await writer.FlushAsync(); } } } } catch (Exception) { writer.WriteEndArray(); throw; } } } } } catch (ObjectDisposedException) { // closing } }