Exemple #1
0
        private long ReadBuildVersion()
        {
            var type = ReadType();

            if (type == null)
            {
                return(0);
            }

            if (type.Equals("BuildVersion", StringComparison.OrdinalIgnoreCase) == false)
            {
                _currentType = GetType(type);
                return(0);
            }

            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.Integer)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected integer BuildVersion, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            return(_state.Long);
        }
            public async ValueTask DisposeAsync()
            {
                if (_disposed)
                {
                    return;
                }

                if (_done == false)
                {
                    while (await MoveNextAsync().ConfigureAwait(false))
                    {
                        // we need to consume the rest of the stream, before we can move next the outer enumerator
                    }
                }

                if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                {
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                }

                if (_state.CurrentTokenType != JsonParserToken.EndObject)
                {
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                }

                _disposed = true;
            }
Exemple #3
0
        private static IEnumerable <GetResponse> ReadResponses(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer)
        {
            if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
            {
                ThrowInvalidJsonResponse(peepingTomStream);
            }

            if (state.CurrentTokenType != JsonParserToken.StartArray)
            {
                ThrowInvalidJsonResponse(peepingTomStream);
            }

            while (true)
            {
                if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                {
                    ThrowInvalidJsonResponse(peepingTomStream);
                }

                if (state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    break;
                }

                yield return(ReadResponse(context, peepingTomStream, parser, state, buffer));
            }
        }
Exemple #4
0
        public static async Task <BlittableJsonReaderObject> SimulateSavingToFileAndLoadingAsync(JsonOperationContext context, BlittableJsonReaderObject toStream)
        {
            //Simulates saving to file and loading
            BlittableJsonReaderObject fromStream;

            await using (Stream stream = new MemoryStream())
            {
                //Pass to stream
                await using (var textWriter = new AsyncBlittableJsonTextWriter(context, stream))
                {
                    context.Write(textWriter, toStream);
                }

                //Get from stream
                stream.Position = 0;

                var state            = new JsonParserState();
                var parser           = new UnmanagedJsonParser(context, state, "some tag");
                var peepingTomStream = new PeepingTomStream(stream, context);

                using (context.GetMemoryBuffer(out var buffer))
                    using (var builder =
                               new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "some tag", parser, state))
                    {
                        await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer);

                        await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer);

                        fromStream = builder.CreateReader();
                    }
            }

            return(fromStream);
        }
Exemple #5
0
            public async Task <bool> MoveNextAsync()
            {
                if (_initialized == false)
                {
                    await InitializeAsync().ConfigureAwait(false);
                }

                if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
                {
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                }

                if (_state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (_state.CurrentTokenType != JsonParserToken.EndObject)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    return(false);
                }

                using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state))
                {
                    await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer).ConfigureAwait(false);

                    Current = builder.CreateReader();
                    return(true);
                }
            }
Exemple #6
0
        public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
        {
            _result       = result;
            _returnBuffer = _context.GetManagedBuffer(out _buffer);
            _state        = new JsonParserState();
            _parser       = new UnmanagedJsonParser(_context, _state, "file");

            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.StartObject)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            buildVersion      = ReadBuildVersion();
            _buildVersionType = BuildVersion.Type(buildVersion);
            _readLegacyEtag   = options.ReadLegacyEtag;

            return(new DisposableAction(() =>
            {
                _parser.Dispose();
                _returnBuffer.Dispose();
                _returnWriteBuffer.Dispose();
            }));
        }
Exemple #7
0
        public IEnumerator <BlittableJsonReaderObject> SetResult(StreamResult response)
        {
            if (response == null)
            {
                throw new InvalidOperationException("The index does not exists, failed to stream results");
            }

            var state = new JsonParserState();

            JsonOperationContext.ManagedPinnedBuffer buffer;

            using (response.Response)
                using (response.Stream)
                    using (var parser = new UnmanagedJsonParser(_session.Context, state, "stream contents"))
                        using (_session.Context.GetManagedBuffer(out buffer))
                            using (var peepingTomStream = new PeepingTomStream(response.Stream, _session.Context))
                            {
                                if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                                {
                                    UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                                }

                                if (state.CurrentTokenType != JsonParserToken.StartObject)
                                {
                                    UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                                }


                                if (_isQueryStream)
                                {
                                    HandleStreamQueryStats(_session.Context, response, parser, state, buffer, _statistics);
                                }

                                var property = UnmanagedJsonParserHelper.ReadString(_session.Context, peepingTomStream, parser, state, buffer);
                                if (string.Equals(property, "Results") == false)
                                {
                                    UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                                }

                                foreach (var result in UnmanagedJsonParserHelper.ReadArrayToMemory(_session.Context, peepingTomStream, parser, state, buffer))
                                {
                                    yield return(result);
                                }

                                if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                                {
                                    UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                                }

                                if (state.CurrentTokenType != JsonParserToken.EndObject)
                                {
                                    UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                                }
                            }
        }
Exemple #8
0
            public async Task <bool> MoveNextAsync()
            {
                if (_initialized == false)
                {
                    await InitializeAsync().ConfigureAwait(false);
                }

                if (_docsCountOnCachedRenewSession <= 16 * 1024)
                {
                    if (_cachedItemsRenew)
                    {
                        _session.Context.CachedProperties = new CachedProperties(_session.Context);
                        ++_docsCountOnCachedRenewSession;
                    }
                }
                else
                {
                    _session.Context.Renew();
                    _docsCountOnCachedRenewSession = 0;
                }

                if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
                {
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                }

                if (_state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (_state.CurrentTokenType != JsonParserToken.EndObject)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    return(false);
                }

                using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state))
                {
                    if (_cachedItemsRenew == false)
                    {
                        _cachedItemsRenew = builder.NeedResetPropertiesCache();
                    }

                    await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer).ConfigureAwait(false);

                    Current = builder.CreateReader();
                    return(true);
                }
            }
Exemple #9
0
        public override void SetResponseRaw(HttpResponseMessage response, Stream stream, JsonOperationContext context)
        {
            var state = new JsonParserState();

            using (var parser = new UnmanagedJsonParser(context, state, "multi_get/response"))
                using (context.GetMemoryBuffer(out var buffer))
                    using (var peepingTomStream = new PeepingTomStream(stream, context))
                        using (_cached)
                        {
                            if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                            {
                                ThrowInvalidJsonResponse(peepingTomStream);
                            }

                            if (state.CurrentTokenType != JsonParserToken.StartObject)
                            {
                                ThrowInvalidJsonResponse(peepingTomStream);
                            }

                            var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
                            if (property != nameof(BlittableArrayResult.Results))
                            {
                                ThrowInvalidJsonResponse(peepingTomStream);
                            }

                            var i = 0;
                            Result = new List <GetResponse>(_commands.Count);
                            foreach (var getResponse in ReadResponses(context, peepingTomStream, parser, state, buffer))
                            {
                                var command = _commands[i];

                                MaybeSetCache(getResponse, command);

                                Result.Add(_cached != null && getResponse.StatusCode == HttpStatusCode.NotModified
                        ? new GetResponse {
                                    Result = _cached.Values[i].Cached.Clone(context), StatusCode = HttpStatusCode.NotModified
                                }
                        : getResponse);

                                i++;
                            }

                            if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                            {
                                ThrowInvalidJsonResponse(peepingTomStream);
                            }

                            if (state.CurrentTokenType != JsonParserToken.EndObject)
                            {
                                ThrowInvalidJsonResponse(peepingTomStream);
                            }
                        }
        }
            public async Task InitializeAsync()
            {
                var property = UnmanagedJsonParserHelper.ReadString(_context, _peepingTomStream, _parser, _state, _buffer);

                if (property.StartsWith(Constants.TimeSeries.QueryFunction) == false)
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                if (_state.CurrentTokenType != JsonParserToken.StartArray)
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
            }
Exemple #11
0
            public async Task InitializeAsync()
            {
                AssertNotSync();

                try
                {
                    _initialized = true;

                    _state        = new JsonParserState();
                    _parser       = new UnmanagedJsonParser(_session.Context, _state, "stream contents");
                    _returnBuffer = _session.Context.GetManagedBuffer(out _buffer);

                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (_state.CurrentTokenType != JsonParserToken.StartObject)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (_isQueryStream)
                    {
                        HandleStreamQueryStats(_session.Context, _response, _parser, _state, _buffer, _streamQueryStatistics);
                    }

                    var property = UnmanagedJsonParserHelper.ReadString(_session.Context, _peepingTomStream, _parser, _state, _buffer);

                    if (string.Equals(property, "Results") == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (_state.CurrentTokenType != JsonParserToken.StartArray)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }
                }
                catch
                {
                    Dispose();

                    throw;
                }
            }
Exemple #12
0
        public override void SetResponseRaw(HttpResponseMessage response, Stream stream, JsonOperationContext context)
        {
            var state = new JsonParserState();

            using (var parser = new UnmanagedJsonParser(context, state, "multi_get/response"))
                using (context.GetManagedBuffer(out JsonOperationContext.ManagedPinnedBuffer buffer))
                    using (var peepingTomStream = new PeepingTomStream(stream, context))
                    {
                        if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                        {
                            ThrowInvalidJsonResponse(peepingTomStream);
                        }

                        if (state.CurrentTokenType != JsonParserToken.StartObject)
                        {
                            ThrowInvalidJsonResponse(peepingTomStream);
                        }

                        var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
                        if (property != nameof(BlittableArrayResult.Results))
                        {
                            ThrowInvalidJsonResponse(peepingTomStream);
                        }

                        var i = 0;
                        Result = new List <GetResponse>();
                        foreach (var getResponse in ReadResponses(context, peepingTomStream, parser, state, buffer))
                        {
                            var command = _commands[i];

                            MaybeSetCache(getResponse, command);
                            MaybeReadFromCache(getResponse, command, context);

                            Result.Add(getResponse);

                            i++;
                        }

                        if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                        {
                            ThrowInvalidJsonResponse(peepingTomStream);
                        }

                        if (state.CurrentTokenType != JsonParserToken.EndObject)
                        {
                            ThrowInvalidJsonResponse(peepingTomStream);
                        }
                    }
        }
            private static void HandleStreamQueryStats(JsonOperationContext context, StreamResult response, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, StreamQueryStatistics streamQueryStatistics = null)
            {
                using (var peepingTomStream = new PeepingTomStream(response.Stream, context))
                {
                    var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
                    if (string.Equals(property, nameof(StreamQueryStatistics.ResultEtag)) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                    var resultEtag = UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer);

                    property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
                    if (string.Equals(property, nameof(StreamQueryStatistics.IsStale)) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);

                    if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);

                    if (state.CurrentTokenType != JsonParserToken.False && state.CurrentTokenType != JsonParserToken.True)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                    var isStale = state.CurrentTokenType != JsonParserToken.False;

                    property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
                    if (string.Equals(property, nameof(StreamQueryStatistics.IndexName)) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                    var indexName = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);

                    property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
                    if (string.Equals(property, nameof(StreamQueryStatistics.TotalResults)) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                    var totalResults = (int)UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer);

                    property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
                    if (string.Equals(property, nameof(StreamQueryStatistics.IndexTimestamp)) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                    var indexTimestamp = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);

                    if (streamQueryStatistics == null)
                        return;

                    streamQueryStatistics.IndexName = indexName;
                    streamQueryStatistics.IsStale = isStale;
                    streamQueryStatistics.TotalResults = totalResults;
                    streamQueryStatistics.ResultEtag = resultEtag;

                    if (DateTime.TryParseExact(indexTimestamp, "o", CultureInfo.InvariantCulture,
                        DateTimeStyles.RoundtripKind, out DateTime timeStamp) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                    streamQueryStatistics.IndexTimestamp = timeStamp;
                }
            }
            public override async Task <ResponseDisposeHandling> ProcessResponse(JsonOperationContext context, HttpCache cache, HttpResponseMessage response, string url)
            {
                var    state  = new JsonParserState();
                Stream stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false);

                using (context.GetMemoryBuffer(out var buffer))
                    using (var parser = new UnmanagedJsonParser(context, state, "attachments/receive"))
                        using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "attachments/list", parser, state))
                            using (var peepingTomStream = new PeepingTomStream(stream, context))
                            {
                                if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
                                {
                                    throw new Exception("cannot parse stream");
                                }

                                if (state.CurrentTokenType != JsonParserToken.StartObject)
                                {
                                    throw new Exception($"Expected token {nameof(JsonParserToken.StartObject)}, but got {nameof(state.CurrentTokenType)}.");
                                }

                                await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);

                                using (var data = builder.CreateReader())
                                {
                                    if (data.TryGetMember(nameof(AttachmentsMetadata), out object obj) && obj is BlittableJsonReaderArray bjra)
                                    {
                                        foreach (BlittableJsonReaderObject e in bjra)
                                        {
                                            var cur = JsonDeserializationClient.AttachmentDetails(e);
                                            AttachmentsMetadata.Add(cur);
                                        }
                                    }
                                }

                                var bufferSize = parser.BufferSize - parser.BufferOffset;
                                var copy       = ArrayPool <byte> .Shared.Rent(bufferSize);

                                var copyMemory = new Memory <byte>(copy);
                                buffer.Memory.Memory.Slice(parser.BufferOffset, bufferSize).CopyTo(copyMemory);

                                Result = Iterate(stream, copy, bufferSize).GetEnumerator();
                            }

                return(ResponseDisposeHandling.Manually);
            }
Exemple #15
0
        private unsafe string ReadType()
        {
            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of object when reading type", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType == JsonParserToken.EndObject)
            {
                return(null);
            }

            if (_state.CurrentTokenType != JsonParserToken.String)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected property type to be string, but was " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            return(_context.AllocateStringValue(null, _state.StringBuffer, _state.StringSize).ToString());
        }
            public async ValueTask<bool> MoveNextAsync()
            {
                if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                if (_state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    _done = true;
                    _current = null;
                    return false;
                }

                using (var builder = new BlittableJsonDocumentBuilder(_context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", _parser, _state))
                {
                    await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false);

                    _current = builder.CreateReader();
                    return true;
                }
            }
            public bool MoveNext()
            {
                AssertInitialized();

                CheckIfContextOrCacheNeedToBeRenewed();

                _timeSeriesIt?.Dispose();

                if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                if (_state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    if (_state.CurrentTokenType != JsonParserToken.EndObject)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    return false;
                }

                _builder.Renew("readArray/singleResult", BlittableJsonDocumentBuilder.UsageMode.ToDisk);

                if (_isTimeSeriesStream)
                    UnmanagedJsonParserHelper.ReadProperty(_builder, _peepingTomStream, _parser, _buffer);
                else
                    UnmanagedJsonParserHelper.ReadObject(_builder, _peepingTomStream, _parser, _buffer);

                Current = _builder.CreateReader();

                _builder.Reset();

                if (_isTimeSeriesStream)
                {
                    _timeSeriesIt = new TimeSeriesStreamEnumerator(_builderContext, _peepingTomStream, _parser, _state, _buffer);
                    _timeSeriesIt.Initialize();
                }

                return true;
            }
            public async ValueTask<bool> MoveNextAsync()
            {
                AssertInitialized();

                CheckIfContextOrCacheNeedToBeRenewed();

                if (_timeSeriesIt != null)
                    await _timeSeriesIt.DisposeAsync().ConfigureAwait(false);

                if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                if (_state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    if (_state.CurrentTokenType != JsonParserToken.EndObject)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    return false;
                }

                _builder.Renew("readArray/singleResult", BlittableJsonDocumentBuilder.UsageMode.ToDisk);

                if (_isTimeSeriesStream)
                    await UnmanagedJsonParserHelper.ReadPropertyAsync(_builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false);
                else
                    await UnmanagedJsonParserHelper.ReadObjectAsync(_builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false);

                Current = _builder.CreateReader();

                _builder.Reset();

                if (_isTimeSeriesStream)
                {
                    _timeSeriesIt = new TimeSeriesStreamEnumerator(_builderContext, _peepingTomStream, _parser, _state, _buffer);
                    await _timeSeriesIt.InitializeAsync().ConfigureAwait(false);
                }
                return true;
            }
            public async Task InitializeAsync()
            {
                AssertNotSync();

                try
                {
                    _initialized = true;

                    _state = new JsonParserState();
                    _parser = new UnmanagedJsonParser(_session.Context, _state, "stream contents");
                    _builder = new BlittableJsonDocumentBuilder(_builderContext, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state);
                    _returnBuffer = _session.Context.GetMemoryBuffer(out _buffer);

                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    if (_state.CurrentTokenType != JsonParserToken.StartObject)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    if (_isQueryStream)
                        HandleStreamQueryStats(_builderContext, _response, _parser, _state, _buffer, _streamQueryStatistics);

                    var property = UnmanagedJsonParserHelper.ReadString(_builderContext, _peepingTomStream, _parser, _state, _buffer);

                    if (string.Equals(property, "Results") == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    if (_state.CurrentTokenType != JsonParserToken.StartArray)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                }
                catch
                {
                    await DisposeAsync().ConfigureAwait(false);

                    throw;
                }
            }
Exemple #20
0
            public bool MoveNext()
            {
                AssertInitialized();

                CheckIfContextNeedsToBeRenewed();

                if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                {
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                }

                if (_state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (_state.CurrentTokenType != JsonParserToken.EndObject)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    return(false);
                }

                using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state))
                {
                    if (_cachedItemsRenew == false)
                    {
                        _cachedItemsRenew = builder.NeedResetPropertiesCache();
                    }

                    UnmanagedJsonParserHelper.ReadObject(builder, _peepingTomStream, _parser, _buffer);

                    Current = builder.CreateReader();
                    return(true);
                }
            }
Exemple #21
0
        private void ReadObject(Action <BlittableJsonReaderObject> readAction)
        {
            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.StartObject)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            using (var builder = CreateBuilder(_context, null))
            {
                _context.CachedProperties.NewDocument();
                ReadObject(builder);

                using (var reader = builder.CreateReader())
                {
                    readAction(reader);
                }
            }
        }
Exemple #22
0
        private async Task ProcessChanges()
        {
            using (_requestExecutor.ContextPool.AllocateOperationContext(out var context))
            {
                while (_cts.IsCancellationRequested == false)
                {
                    var state = new JsonParserState();

                    using (var stream = new WebSocketStream(_client, _cts.Token))
                        using (context.GetManagedBuffer(out var buffer))
                            using (var parser = new UnmanagedJsonParser(context, state, "changes/receive"))
                                using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", parser, state))
                                    using (var peepingTomStream = new PeepingTomStream(stream, context))
                                    {
                                        if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
                                        {
                                            continue;
                                        }

                                        if (state.CurrentTokenType != JsonParserToken.StartArray)
                                        {
                                            continue;
                                        }

                                        while (true)
                                        {
                                            if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
                                            {
                                                continue;
                                            }

                                            if (state.CurrentTokenType == JsonParserToken.EndArray)
                                            {
                                                break;
                                            }

                                            builder.Renew("changes/receive", BlittableJsonDocumentBuilder.UsageMode.None);

                                            await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);

                                            var json = builder.CreateReader();

                                            try
                                            {
                                                if (json.TryGet("Type", out string type) == false)
                                                {
                                                    continue;
                                                }

                                                switch (type)
                                                {
                                                case "Error":
                                                    json.TryGet("Exception", out string exceptionAsString);
                                                    NotifyAboutError(new Exception(exceptionAsString));
                                                    break;

                                                case "Confirm":
                                                    if (json.TryGet("CommandId", out int commandId) &&
                                                        _confirmations.TryRemove(commandId, out var tcs))
                                                    {
                                                        tcs.TrySetResult(null);
                                                    }
                                                    break;

                                                default:
                                                    json.TryGet("Value", out BlittableJsonReaderObject value);
                                                    NotifySubscribers(type, value, _counters.ValuesSnapshot);
                                                    break;
                                                }
                                            }
                                            catch (Exception e)
                                            {
                                                NotifyAboutError(e);
                                                throw new ChangeProcessingException(e);
                                            }
                                        }
                                    }
                }
            }
        }
        public async Task ConnectInternal()
        {
            try
            {
                var urlBuilder = new StringBuilder(_url).Append("/admin/traffic-watch");

                if (string.IsNullOrWhiteSpace(_database) == false)
                {
                    urlBuilder.Append("?resourceName=").Append(_database);
                }

                var stringUrl = ToWebSocketPath(urlBuilder.ToString().ToLower());
                var url       = new Uri(stringUrl, UriKind.Absolute);
                _client = new ClientWebSocket();
                if (_cert != null)
                {
                    _client.Options.ClientCertificates.Add(_cert);
                }

                await _client.ConnectAsync(url, _cancellationTokenSource.Token).ConfigureAwait(false);

                _firstConnection = false;

                Console.WriteLine($"Connected to RavenDB server. Collecting traffic watch entries to {_path}");

                const int maxFileSize = 128 * 1024 * 1024;
                while (_cancellationTokenSource.IsCancellationRequested == false)
                {
                    string file = Path.Combine(_path, _logNameCreator.GetNewFileName());

                    var state = new JsonParserState();

                    using (var context = JsonOperationContext.ShortTermSingleUse())
                        // Read
                        await using (var stream = new WebSocketStream(_client, _cancellationTokenSource.Token))
                            using (context.GetMemoryBuffer(out var buffer))
                                using (var parser = new UnmanagedJsonParser(context, state, "trafficwatch/receive"))
                                    using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readObject/singleResult", parser, state))
                                        // Write
                                        await using (var fileStream = new FileStream(file, FileMode.Append, FileAccess.Write, FileShare.Read, 32 * 1024, false))
                                            await using (var gZipStream = new GZipStream(fileStream, CompressionMode.Compress, false))
                                                using (var peepingTomStream = new PeepingTomStream(stream, context))
                                                    await using (var writer = new AsyncBlittableJsonTextWriter(context, gZipStream))
                                                    {
                                                        writer.WriteStartArray();
                                                        var isFirst = true;

                                                        while (fileStream.Length < maxFileSize)
                                                        {
                                                            if (_cancellationTokenSource.IsCancellationRequested)
                                                            {
                                                                writer.WriteEndArray();
                                                                break;
                                                            }

                                                            try
                                                            {
                                                                var flushCount = 0;
                                                                while (fileStream.Length < maxFileSize && _cancellationTokenSource.IsCancellationRequested == false)
                                                                {
                                                                    builder.Reset();
                                                                    builder.Renew("trafficwatch/receive", BlittableJsonDocumentBuilder.UsageMode.None);

                                                                    if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
                                                                    {
                                                                        continue;
                                                                    }

                                                                    await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);

                                                                    using (var json = builder.CreateReader())
                                                                    {
                                                                        if (_changeTypes != null)
                                                                        {
                                                                            if (json.TryGet("Type", out TrafficWatchChangeType type) == false)
                                                                            {
                                                                                continue;
                                                                            }
                                                                            if (_changeTypes.Contains(type) == false)
                                                                            {
                                                                                continue;
                                                                            }
                                                                        }

                                                                        if (_database != null)
                                                                        {
                                                                            if (json.TryGet("DatabaseName", out LazyStringValue databaseName) == false ||
                                                                                _database.Equals(databaseName, StringComparison.OrdinalIgnoreCase) == false)
                                                                            {
                                                                                continue;
                                                                            }
                                                                        }

                                                                        if (isFirst == false)
                                                                        {
                                                                            writer.WriteComma();
                                                                        }

                                                                        isFirst = false;
                                                                        if (_verbose)
                                                                        {
                                                                            Console.WriteLine(json);
                                                                        }

                                                                        writer.WriteObject(json);
                                                                        _errorCount = 0;
                                                                        if (flushCount++ % 128 == 0)
                                                                        {
                                                                            await writer.FlushAsync();
                                                                        }
                                                                    }
                                                                }
                                                            }
                                                            catch (Exception)
                                                            {
                                                                writer.WriteEndArray();
                                                                throw;
                                                            }
                                                        }
                                                    }
                }
            }
            catch (ObjectDisposedException)
            {
                // closing
            }
        }
        internal static IEnumerable <ReplayProgress> Replay(DocumentDatabase database, Stream replayStream)
        {
            DocumentsOperationContext txCtx   = null;
            IDisposable          txDisposable = null;
            DocumentsTransaction previousTx   = null;

            using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                using (context.GetManagedBuffer(out var buffer))
                    using (var gZipStream = new GZipStream(replayStream, CompressionMode.Decompress, leaveOpen: true))
                    {
                        var peepingTomStream = new PeepingTomStream(gZipStream, context);
                        var state            = new JsonParserState();
                        var parser           = new UnmanagedJsonParser(context, state, "file");

                        var commandsProgress = 0;
                        var readers          = UnmanagedJsonParserHelper.ReadArrayToMemory(context, peepingTomStream, parser, state, buffer);
                        using (var readersItr = readers.GetEnumerator())
                        {
                            ReadStartRecordingDetails(readersItr, context, peepingTomStream);
                            while (readersItr.MoveNext())
                            {
                                using (readersItr.Current)
                                {
                                    if (readersItr.Current.TryGet(nameof(RecordingDetails.Type), out string strType) == false)
                                    {
                                        throw new ReplayTransactionsException($"Can't read {nameof(RecordingDetails.Type)} of replay detail", peepingTomStream);
                                    }

                                    if (Enum.TryParse <TxInstruction>(strType, true, out var type))
                                    {
                                        switch (type)
                                        {
                                        case TxInstruction.BeginTx:
                                            txDisposable = database.DocumentsStorage.ContextPool.AllocateOperationContext(out txCtx);
                                            txCtx.OpenWriteTransaction();
                                            break;

                                        case TxInstruction.Commit:
                                            txCtx.Transaction.Commit();
                                            break;

                                        case TxInstruction.DisposeTx:
                                            txDisposable.Dispose();
                                            break;

                                        case TxInstruction.BeginAsyncCommitAndStartNewTransaction:
                                            previousTx        = txCtx.Transaction;
                                            txCtx.Transaction = txCtx.Transaction.BeginAsyncCommitAndStartNewTransaction(txCtx);
                                            txDisposable      = txCtx.Transaction;
                                            break;

                                        case TxInstruction.EndAsyncCommit:
                                            previousTx.EndAsyncCommit();
                                            break;

                                        case TxInstruction.DisposePrevTx:
                                            previousTx.Dispose();
                                            break;
                                        }
                                        continue;
                                    }

                                    try
                                    {
                                        var cmd = DeserializeCommand(context, database, strType, readersItr.Current, peepingTomStream);
                                        commandsProgress += cmd.ExecuteDirectly(txCtx);
                                        TransactionOperationsMerger.UpdateGlobalReplicationInfoBeforeCommit(txCtx);
                                    }
                                    catch (Exception)
                                    {
                                        //TODO To accept exceptions that was thrown while recording
                                        txDisposable.Dispose();
                                        throw;
                                    }

                                    yield return(new ReplayProgress
                                    {
                                        CommandsProgress = commandsProgress
                                    });
                                }
                            }
                        }
                    }
        }
Exemple #25
0
        private static unsafe GetResponse ReadResponse(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer)
        {
            if (state.CurrentTokenType != JsonParserToken.StartObject)
            {
                ThrowInvalidJsonResponse(peepingTomStream);
            }

            var getResponse = new GetResponse();

            while (true)
            {
                if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                {
                    ThrowInvalidJsonResponse(peepingTomStream);
                }

                if (state.CurrentTokenType == JsonParserToken.EndObject)
                {
                    break;
                }

                if (state.CurrentTokenType != JsonParserToken.String)
                {
                    ThrowInvalidJsonResponse(peepingTomStream);
                }

                var property = context.AllocateStringValue(null, state.StringBuffer, state.StringSize).ToString();
                switch (property)
                {
                case nameof(GetResponse.Result):
                    if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                    {
                        ThrowInvalidJsonResponse(peepingTomStream);
                    }

                    if (state.CurrentTokenType == JsonParserToken.Null)
                    {
                        continue;
                    }

                    if (state.CurrentTokenType != JsonParserToken.StartObject)
                    {
                        ThrowInvalidJsonResponse(peepingTomStream);
                    }

                    using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "multi_get/result", parser, state))
                    {
                        UnmanagedJsonParserHelper.ReadObject(builder, peepingTomStream, parser, buffer);
                        getResponse.Result = builder.CreateReader();
                    }
                    continue;

                case nameof(GetResponse.Headers):
                    if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                    {
                        ThrowInvalidJsonResponse(peepingTomStream);
                    }

                    if (state.CurrentTokenType == JsonParserToken.Null)
                    {
                        continue;
                    }

                    if (state.CurrentTokenType != JsonParserToken.StartObject)
                    {
                        ThrowInvalidJsonResponse(peepingTomStream);
                    }

                    using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "multi_get/result", parser, state))
                    {
                        UnmanagedJsonParserHelper.ReadObject(builder, peepingTomStream, parser, buffer);
                        using (var headersJson = builder.CreateReader())
                        {
                            foreach (var propertyName in headersJson.GetPropertyNames())
                            {
                                getResponse.Headers[propertyName] = headersJson[propertyName].ToString();
                            }
                        }
                    }
                    continue;

                case nameof(GetResponse.StatusCode):
                    if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                    {
                        ThrowInvalidJsonResponse(peepingTomStream);
                    }

                    if (state.CurrentTokenType != JsonParserToken.Integer)
                    {
                        ThrowInvalidJsonResponse(peepingTomStream);
                    }

                    getResponse.StatusCode = (HttpStatusCode)state.Long;
                    continue;

                default:
                    ThrowInvalidJsonResponse(peepingTomStream);
                    break;
                }
            }

            return(getResponse);
        }
Exemple #26
0
        private IEnumerable <DocumentConflict> ReadConflicts(INewDocumentActions actions = null)
        {
            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.StartArray)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            var context = _context;
            var builder = CreateBuilder(context, null);

            try
            {
                while (true)
                {
                    if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
                    }

                    if (_state.CurrentTokenType == JsonParserToken.EndArray)
                    {
                        break;
                    }

                    if (actions != null)
                    {
                        var oldContext = context;
                        context = actions.GetContextForNewDocument();
                        if (oldContext != context)
                        {
                            builder.Dispose();
                            builder = CreateBuilder(context, null);
                        }
                    }
                    builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);

                    _context.CachedProperties.NewDocument();

                    ReadObject(builder);

                    var data = builder.CreateReader();
                    builder.Reset();

                    var conflict = new DocumentConflict();
                    if (data.TryGet(nameof(DocumentConflict.Id), out conflict.Id) &&
                        data.TryGet(nameof(DocumentConflict.Collection), out conflict.Collection) &&
                        data.TryGet(nameof(DocumentConflict.Flags), out string flags) &&
                        data.TryGet(nameof(DocumentConflict.ChangeVector), out conflict.ChangeVector) &&
                        data.TryGet(nameof(DocumentConflict.Etag), out conflict.Etag) &&
                        data.TryGet(nameof(DocumentConflict.LastModified), out conflict.LastModified) &&
                        data.TryGet(nameof(DocumentConflict.Doc), out conflict.Doc))
                    {
                        conflict.Flags = Enum.Parse <DocumentFlags>(flags);
                        if (conflict.Doc != null) // This is null for conflict that was generated from tombstone
                        {
                            conflict.Doc = context.ReadObject(conflict.Doc, conflict.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
                        }
                        yield return(conflict);
                    }
                    else
                    {
                        var msg = "Ignoring an invalid conflict which you try to import. " + data;
                        if (_log.IsOperationsEnabled)
                        {
                            _log.Operations(msg);
                        }

                        _result.Conflicts.ErroredCount++;
                        _result.AddWarning(msg);
                    }
                }
            }
            finally
            {
                builder.Dispose();
            }
        }
Exemple #27
0
        private IEnumerable <Tombstone> ReadTombstones(INewDocumentActions actions = null)
        {
            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.StartArray)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            var context = _context;
            var builder = CreateBuilder(context, null);

            try
            {
                while (true)
                {
                    if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
                    }

                    if (_state.CurrentTokenType == JsonParserToken.EndArray)
                    {
                        break;
                    }

                    if (actions != null)
                    {
                        var oldContext = context;
                        context = actions.GetContextForNewDocument();
                        if (oldContext != context)
                        {
                            builder.Dispose();
                            builder = CreateBuilder(context, null);
                        }
                    }
                    builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);

                    _context.CachedProperties.NewDocument();

                    ReadObject(builder);

                    var data = builder.CreateReader();
                    builder.Reset();

                    var tombstone = new Tombstone();
                    if (data.TryGet("Key", out tombstone.LowerId) &&
                        data.TryGet(nameof(Tombstone.Type), out string type) &&
                        data.TryGet(nameof(Tombstone.Collection), out tombstone.Collection) &&
                        data.TryGet(nameof(Tombstone.LastModified), out tombstone.LastModified))
                    {
                        if (Enum.TryParse <Tombstone.TombstoneType>(type, out var tombstoneType) == false)
                        {
                            var msg = $"Ignoring a tombstone of type `{type}` which is not supported in 4.0. ";
                            if (_log.IsOperationsEnabled)
                            {
                                _log.Operations(msg);
                            }

                            _result.Tombstones.ErroredCount++;
                            _result.AddWarning(msg);
                            continue;
                        }

                        tombstone.Type = tombstoneType;
                        yield return(tombstone);
                    }
                    else
                    {
                        var msg = "Ignoring an invalid tombstone which you try to import. " + data;
                        if (_log.IsOperationsEnabled)
                        {
                            _log.Operations(msg);
                        }

                        _result.Tombstones.ErroredCount++;
                        _result.AddWarning(msg);
                    }
                }
            }
            finally
            {
                builder.Dispose();
            }
        }
        public void JsonDeserialize_WhenHasBlittableObjectPropertyAndWriteAndReadFromStream_ShouldResultInCommandWithTheProperty()
        {
            using (Server.ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
            {
                var jsonSerializer = new JsonSerializer
                {
                    ContractResolver = new DefaultRavenContractResolver(DocumentConventions.Default.Serialization),
                };
                jsonSerializer.Converters.Add(BlittableJsonConverter.Instance);

                var data     = new { SomeProperty = "SomeValue" };
                var expected = DocumentConventions.Default.Serialization.DefaultConverter.ToBlittable(data, context);
                var command  = new Command {
                    BlittableObject = expected
                };

                //Serialize
                BlittableJsonReaderObject toStream;
                using (var writer = new BlittableJsonWriter(context))
                {
                    jsonSerializer.Serialize(writer, command);
                    writer.FinalizeDocument();

                    toStream = writer.CreateReader();
                }

                //Simulates copying to file and loading
                BlittableJsonReaderObject fromStream;
                using (Stream stream = new MemoryStream())
                {
                    //Pass to stream
                    using (var textWriter = new BlittableJsonTextWriter(context, stream))
                    {
                        context.Write(textWriter, toStream);
                    }

                    //Get from stream
                    stream.Position = 0;

                    var state            = new JsonParserState();
                    var parser           = new UnmanagedJsonParser(context, state, "some tag");
                    var peepingTomStream = new PeepingTomStream(stream, context);

                    using (context.GetMemoryBuffer(out var buffer))
                        using (var builder =
                                   new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "some tag", parser, state))
                        {
                            UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer);
                            UnmanagedJsonParserHelper.ReadObject(builder, peepingTomStream, parser, buffer);

                            fromStream = builder.CreateReader();
                        }
                }

                //Deserialize
                BlittableJsonReaderObject actual;
                using (var reader = new BlittableJsonReader(context))
                {
                    reader.Initialize(fromStream);
                    var deserialized = jsonSerializer.Deserialize <Command>(reader);
                    actual = deserialized.BlittableObject;
                }

                Assert.Equal(expected, actual);
            }
        }
Exemple #29
0
        private async Task ProcessChanges()
        {
            using (_requestExecutor.ContextPool.AllocateOperationContext(out var context))
            {
                while (_cts.IsCancellationRequested == false)
                {
                    context.Reset();
                    context.Renew();

                    var state = new JsonParserState();

                    using (var stream = new WebSocketStream(_client, _cts.Token))
                        using (context.GetMemoryBuffer(out JsonOperationContext.MemoryBuffer buffer))
                            using (var parser = new UnmanagedJsonParser(context, state, "changes/receive"))
                                using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", parser, state))
                                    using (var peepingTomStream = new PeepingTomStream(stream, context))
                                    {
                                        if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
                                        {
                                            continue;
                                        }

                                        if (state.CurrentTokenType != JsonParserToken.StartArray)
                                        {
                                            continue;
                                        }

                                        while (true)
                                        {
                                            builder.Reset();
                                            builder.Renew("changes/receive", BlittableJsonDocumentBuilder.UsageMode.None);

                                            if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
                                            {
                                                continue;
                                            }

                                            if (state.CurrentTokenType == JsonParserToken.EndArray)
                                            {
                                                break;
                                            }

                                            await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);

                                            using (var json = builder.CreateReader())
                                            {
                                                try
                                                {
                                                    if (json.TryGet(nameof(TopologyChange), out bool supports) && supports)
                                                    {
                                                        GetOrAddConnectionState("Topology", "watch-topology-change", "", "");
                                                        await _requestExecutor.UpdateTopologyAsync(new RequestExecutor.UpdateTopologyParameters(_serverNode)
                                                        {
                                                            TimeoutInMs = 0, ForceUpdate = true, DebugTag = "watch-topology-change"
                                                        }).ConfigureAwait(false);

                                                        continue;
                                                    }

                                                    if (json.TryGet("Type", out string type) == false)
                                                    {
                                                        continue;
                                                    }

                                                    switch (type)
                                                    {
                                                    case "Error":
                                                        json.TryGet("Exception", out string exceptionAsString);
                                                        NotifyAboutError(new Exception(exceptionAsString));
                                                        break;

                                                    case "Confirm":
                                                        if (json.TryGet("CommandId", out int commandId) &&
                                                            _confirmations.TryRemove(commandId, out var tcs))
                                                        {
                                                            tcs.TrySetResult(null);
                                                        }

                                                        break;

                                                    default:
                                                        json.TryGet("Value", out BlittableJsonReaderObject value);
                                                        NotifySubscribers(type, value, _counters.ForceEnumerateInThreadSafeManner().Select(x => x.Value).ToList());
                                                        break;
                                                    }
                                                }
                                                catch (Exception e)
                                                {
                                                    NotifyAboutError(e);
                                                    throw new ChangeProcessingException(e);
                                                }
                                            }
                                        }
                                    }
                }
            }
        }
Exemple #30
0
        private IEnumerable <DocumentItem> ReadDocuments(INewDocumentActions actions = null)
        {
            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.StartArray)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            var context      = _context;
            var legacyImport = _buildVersionType == BuildVersionType.V3;
            var modifier     = new BlittableMetadataModifier(context)
            {
                ReadFirstEtagOfLegacyRevision = legacyImport,
                ReadLegacyEtag = _readLegacyEtag,
                OperateOnTypes = _operateOnTypes
            };
            var builder = CreateBuilder(context, modifier);

            try
            {
                List <DocumentItem.AttachmentStream> attachments = null;
                while (true)
                {
                    if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
                    }

                    if (_state.CurrentTokenType == JsonParserToken.EndArray)
                    {
                        break;
                    }

                    if (actions != null)
                    {
                        var oldContext = context;
                        context = actions.GetContextForNewDocument();
                        if (oldContext != context)
                        {
                            builder.Dispose();
                            builder = CreateBuilder(context, modifier);
                        }
                    }
                    builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);

                    _context.CachedProperties.NewDocument();

                    ReadObject(builder);

                    var data = builder.CreateReader();
                    builder.Reset();

                    if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
                        metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type))
                    {
                        if (type != DocumentItem.ExportDocumentType.Attachment)
                        {
                            var msg = $"Ignoring an item of type `{type}`. " + data;
                            if (_log.IsOperationsEnabled)
                            {
                                _log.Operations(msg);
                            }
                            _result.AddWarning(msg);
                            continue;
                        }

                        if (attachments == null)
                        {
                            attachments = new List <DocumentItem.AttachmentStream>();
                        }

                        var attachment = new DocumentItem.AttachmentStream
                        {
                            Stream = actions.GetTempStream()
                        };
                        ProcessAttachmentStream(context, data, ref attachment);
                        attachments.Add(attachment);
                        continue;
                    }

                    if (legacyImport)
                    {
                        if (modifier.Id.Contains(HiLoHandler.RavenHiloIdPrefix))
                        {
                            data.Modifications = new DynamicJsonValue
                            {
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection
                                }
                            };
                        }
                    }

                    if (data.Modifications != null)
                    {
                        data = context.ReadObject(data, modifier.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
                    }

                    _result.LegacyLastDocumentEtag = modifier.LegacyEtag;

                    yield return(new DocumentItem
                    {
                        Document = new Document
                        {
                            Data = data,
                            Id = modifier.Id,
                            ChangeVector = modifier.ChangeVector,
                            Flags = modifier.Flags,
                            NonPersistentFlags = modifier.NonPersistentFlags,
                            LastModified = modifier.LastModified ?? _database.Time.GetUtcNow(),
                        },
                        Attachments = attachments
                    });

                    attachments = null;
                }
            }
            finally
            {
                builder.Dispose();
            }
        }