Exemple #1
0
        public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
        {
            _result       = result;
            _returnBuffer = _context.GetManagedBuffer(out _buffer);
            _state        = new JsonParserState();
            _parser       = new UnmanagedJsonParser(_context, _state, "file");

            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.StartObject)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            buildVersion      = ReadBuildVersion();
            _buildVersionType = BuildVersion.Type(buildVersion);
            _readLegacyEtag   = options.ReadLegacyEtag;

            return(new DisposableAction(() =>
            {
                _parser.Dispose();
                _returnBuffer.Dispose();
                _returnWriteBuffer.Dispose();
            }));
        }
            public async ValueTask DisposeAsync()
            {
                if (_disposed)
                {
                    return;
                }

                if (_done == false)
                {
                    while (await MoveNextAsync().ConfigureAwait(false))
                    {
                        // we need to consume the rest of the stream, before we can move next the outer enumerator
                    }
                }

                if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                {
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                }

                if (_state.CurrentTokenType != JsonParserToken.EndObject)
                {
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                }

                _disposed = true;
            }
Exemple #3
0
        private long ReadBuildVersion()
        {
            var type = ReadType();

            if (type == null)
            {
                return(0);
            }

            if (type.Equals("BuildVersion", StringComparison.OrdinalIgnoreCase) == false)
            {
                _currentType = GetType(type);
                return(0);
            }

            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.Integer)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected integer BuildVersion, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            return(_state.Long);
        }
Exemple #4
0
            public async Task <bool> MoveNextAsync()
            {
                if (_initialized == false)
                {
                    await InitializeAsync().ConfigureAwait(false);
                }

                if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
                {
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                }

                if (_state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (_state.CurrentTokenType != JsonParserToken.EndObject)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    return(false);
                }

                using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state))
                {
                    await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer).ConfigureAwait(false);

                    Current = builder.CreateReader();
                    return(true);
                }
            }
Exemple #5
0
        public IEnumerator <BlittableJsonReaderObject> SetResult(StreamResult response)
        {
            if (response == null)
            {
                throw new InvalidOperationException("The index does not exists, failed to stream results");
            }

            var state = new JsonParserState();

            JsonOperationContext.ManagedPinnedBuffer buffer;

            using (response.Response)
                using (response.Stream)
                    using (var parser = new UnmanagedJsonParser(_session.Context, state, "stream contents"))
                        using (_session.Context.GetManagedBuffer(out buffer))
                            using (var peepingTomStream = new PeepingTomStream(response.Stream, _session.Context))
                            {
                                if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                                {
                                    UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                                }

                                if (state.CurrentTokenType != JsonParserToken.StartObject)
                                {
                                    UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                                }


                                if (_isQueryStream)
                                {
                                    HandleStreamQueryStats(_session.Context, response, parser, state, buffer, _statistics);
                                }

                                var property = UnmanagedJsonParserHelper.ReadString(_session.Context, peepingTomStream, parser, state, buffer);
                                if (string.Equals(property, "Results") == false)
                                {
                                    UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                                }

                                foreach (var result in UnmanagedJsonParserHelper.ReadArrayToMemory(_session.Context, peepingTomStream, parser, state, buffer))
                                {
                                    yield return(result);
                                }

                                if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                                {
                                    UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                                }

                                if (state.CurrentTokenType != JsonParserToken.EndObject)
                                {
                                    UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                                }
                            }
        }
Exemple #6
0
            public async Task <bool> MoveNextAsync()
            {
                if (_initialized == false)
                {
                    await InitializeAsync().ConfigureAwait(false);
                }

                if (_docsCountOnCachedRenewSession <= 16 * 1024)
                {
                    if (_cachedItemsRenew)
                    {
                        _session.Context.CachedProperties = new CachedProperties(_session.Context);
                        ++_docsCountOnCachedRenewSession;
                    }
                }
                else
                {
                    _session.Context.Renew();
                    _docsCountOnCachedRenewSession = 0;
                }

                if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
                {
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                }

                if (_state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (_state.CurrentTokenType != JsonParserToken.EndObject)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    return(false);
                }

                using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state))
                {
                    if (_cachedItemsRenew == false)
                    {
                        _cachedItemsRenew = builder.NeedResetPropertiesCache();
                    }

                    await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer).ConfigureAwait(false);

                    Current = builder.CreateReader();
                    return(true);
                }
            }
            public async Task InitializeAsync()
            {
                var property = UnmanagedJsonParserHelper.ReadString(_context, _peepingTomStream, _parser, _state, _buffer);

                if (property.StartsWith(Constants.TimeSeries.QueryFunction) == false)
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                if (_state.CurrentTokenType != JsonParserToken.StartArray)
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
            }
Exemple #8
0
            public async Task InitializeAsync()
            {
                AssertNotSync();

                try
                {
                    _initialized = true;

                    _state        = new JsonParserState();
                    _parser       = new UnmanagedJsonParser(_session.Context, _state, "stream contents");
                    _returnBuffer = _session.Context.GetManagedBuffer(out _buffer);

                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (_state.CurrentTokenType != JsonParserToken.StartObject)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (_isQueryStream)
                    {
                        HandleStreamQueryStats(_session.Context, _response, _parser, _state, _buffer, _streamQueryStatistics);
                    }

                    var property = UnmanagedJsonParserHelper.ReadString(_session.Context, _peepingTomStream, _parser, _state, _buffer);

                    if (string.Equals(property, "Results") == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (_state.CurrentTokenType != JsonParserToken.StartArray)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }
                }
                catch
                {
                    Dispose();

                    throw;
                }
            }
            private static void HandleStreamQueryStats(JsonOperationContext context, StreamResult response, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, StreamQueryStatistics streamQueryStatistics = null)
            {
                using (var peepingTomStream = new PeepingTomStream(response.Stream, context))
                {
                    var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
                    if (string.Equals(property, nameof(StreamQueryStatistics.ResultEtag)) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                    var resultEtag = UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer);

                    property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
                    if (string.Equals(property, nameof(StreamQueryStatistics.IsStale)) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);

                    if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);

                    if (state.CurrentTokenType != JsonParserToken.False && state.CurrentTokenType != JsonParserToken.True)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                    var isStale = state.CurrentTokenType != JsonParserToken.False;

                    property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
                    if (string.Equals(property, nameof(StreamQueryStatistics.IndexName)) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                    var indexName = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);

                    property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
                    if (string.Equals(property, nameof(StreamQueryStatistics.TotalResults)) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                    var totalResults = (int)UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer);

                    property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
                    if (string.Equals(property, nameof(StreamQueryStatistics.IndexTimestamp)) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                    var indexTimestamp = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);

                    if (streamQueryStatistics == null)
                        return;

                    streamQueryStatistics.IndexName = indexName;
                    streamQueryStatistics.IsStale = isStale;
                    streamQueryStatistics.TotalResults = totalResults;
                    streamQueryStatistics.ResultEtag = resultEtag;

                    if (DateTime.TryParseExact(indexTimestamp, "o", CultureInfo.InvariantCulture,
                        DateTimeStyles.RoundtripKind, out DateTime timeStamp) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
                    streamQueryStatistics.IndexTimestamp = timeStamp;
                }
            }
Exemple #10
0
        private unsafe string ReadType()
        {
            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of object when reading type", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType == JsonParserToken.EndObject)
            {
                return(null);
            }

            if (_state.CurrentTokenType != JsonParserToken.String)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected property type to be string, but was " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            return(_context.AllocateStringValue(null, _state.StringBuffer, _state.StringSize).ToString());
        }
            public async ValueTask<bool> MoveNextAsync()
            {
                if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                if (_state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    _done = true;
                    _current = null;
                    return false;
                }

                using (var builder = new BlittableJsonDocumentBuilder(_context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", _parser, _state))
                {
                    await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false);

                    _current = builder.CreateReader();
                    return true;
                }
            }
            public bool MoveNext()
            {
                AssertInitialized();

                CheckIfContextOrCacheNeedToBeRenewed();

                _timeSeriesIt?.Dispose();

                if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                if (_state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    if (_state.CurrentTokenType != JsonParserToken.EndObject)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    return false;
                }

                _builder.Renew("readArray/singleResult", BlittableJsonDocumentBuilder.UsageMode.ToDisk);

                if (_isTimeSeriesStream)
                    UnmanagedJsonParserHelper.ReadProperty(_builder, _peepingTomStream, _parser, _buffer);
                else
                    UnmanagedJsonParserHelper.ReadObject(_builder, _peepingTomStream, _parser, _buffer);

                Current = _builder.CreateReader();

                _builder.Reset();

                if (_isTimeSeriesStream)
                {
                    _timeSeriesIt = new TimeSeriesStreamEnumerator(_builderContext, _peepingTomStream, _parser, _state, _buffer);
                    _timeSeriesIt.Initialize();
                }

                return true;
            }
            public async ValueTask<bool> MoveNextAsync()
            {
                AssertInitialized();

                CheckIfContextOrCacheNeedToBeRenewed();

                if (_timeSeriesIt != null)
                    await _timeSeriesIt.DisposeAsync().ConfigureAwait(false);

                if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                if (_state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    if (_state.CurrentTokenType != JsonParserToken.EndObject)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    return false;
                }

                _builder.Renew("readArray/singleResult", BlittableJsonDocumentBuilder.UsageMode.ToDisk);

                if (_isTimeSeriesStream)
                    await UnmanagedJsonParserHelper.ReadPropertyAsync(_builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false);
                else
                    await UnmanagedJsonParserHelper.ReadObjectAsync(_builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false);

                Current = _builder.CreateReader();

                _builder.Reset();

                if (_isTimeSeriesStream)
                {
                    _timeSeriesIt = new TimeSeriesStreamEnumerator(_builderContext, _peepingTomStream, _parser, _state, _buffer);
                    await _timeSeriesIt.InitializeAsync().ConfigureAwait(false);
                }
                return true;
            }
            public async Task InitializeAsync()
            {
                AssertNotSync();

                try
                {
                    _initialized = true;

                    _state = new JsonParserState();
                    _parser = new UnmanagedJsonParser(_session.Context, _state, "stream contents");
                    _builder = new BlittableJsonDocumentBuilder(_builderContext, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state);
                    _returnBuffer = _session.Context.GetMemoryBuffer(out _buffer);

                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    if (_state.CurrentTokenType != JsonParserToken.StartObject)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    if (_isQueryStream)
                        HandleStreamQueryStats(_builderContext, _response, _parser, _state, _buffer, _streamQueryStatistics);

                    var property = UnmanagedJsonParserHelper.ReadString(_builderContext, _peepingTomStream, _parser, _state, _buffer);

                    if (string.Equals(property, "Results") == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);

                    if (_state.CurrentTokenType != JsonParserToken.StartArray)
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                }
                catch
                {
                    await DisposeAsync().ConfigureAwait(false);

                    throw;
                }
            }
Exemple #15
0
            public bool MoveNext()
            {
                AssertInitialized();

                CheckIfContextNeedsToBeRenewed();

                if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                {
                    UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                }

                if (_state.CurrentTokenType == JsonParserToken.EndArray)
                {
                    if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    if (_state.CurrentTokenType != JsonParserToken.EndObject)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
                    }

                    return(false);
                }

                using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state))
                {
                    if (_cachedItemsRenew == false)
                    {
                        _cachedItemsRenew = builder.NeedResetPropertiesCache();
                    }

                    UnmanagedJsonParserHelper.ReadObject(builder, _peepingTomStream, _parser, _buffer);

                    Current = builder.CreateReader();
                    return(true);
                }
            }
Exemple #16
0
        private void ReadObject(Action <BlittableJsonReaderObject> readAction)
        {
            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.StartObject)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            using (var builder = CreateBuilder(_context, null))
            {
                _context.CachedProperties.NewDocument();
                ReadObject(builder);

                using (var reader = builder.CreateReader())
                {
                    readAction(reader);
                }
            }
        }
Exemple #17
0
        private IEnumerable <DocumentConflict> ReadConflicts(INewDocumentActions actions = null)
        {
            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.StartArray)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            var context = _context;
            var builder = CreateBuilder(context, null);

            try
            {
                while (true)
                {
                    if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
                    }

                    if (_state.CurrentTokenType == JsonParserToken.EndArray)
                    {
                        break;
                    }

                    if (actions != null)
                    {
                        var oldContext = context;
                        context = actions.GetContextForNewDocument();
                        if (oldContext != context)
                        {
                            builder.Dispose();
                            builder = CreateBuilder(context, null);
                        }
                    }
                    builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);

                    _context.CachedProperties.NewDocument();

                    ReadObject(builder);

                    var data = builder.CreateReader();
                    builder.Reset();

                    var conflict = new DocumentConflict();
                    if (data.TryGet(nameof(DocumentConflict.Id), out conflict.Id) &&
                        data.TryGet(nameof(DocumentConflict.Collection), out conflict.Collection) &&
                        data.TryGet(nameof(DocumentConflict.Flags), out string flags) &&
                        data.TryGet(nameof(DocumentConflict.ChangeVector), out conflict.ChangeVector) &&
                        data.TryGet(nameof(DocumentConflict.Etag), out conflict.Etag) &&
                        data.TryGet(nameof(DocumentConflict.LastModified), out conflict.LastModified) &&
                        data.TryGet(nameof(DocumentConflict.Doc), out conflict.Doc))
                    {
                        conflict.Flags = Enum.Parse <DocumentFlags>(flags);
                        if (conflict.Doc != null) // This is null for conflict that was generated from tombstone
                        {
                            conflict.Doc = context.ReadObject(conflict.Doc, conflict.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
                        }
                        yield return(conflict);
                    }
                    else
                    {
                        var msg = "Ignoring an invalid conflict which you try to import. " + data;
                        if (_log.IsOperationsEnabled)
                        {
                            _log.Operations(msg);
                        }

                        _result.Conflicts.ErroredCount++;
                        _result.AddWarning(msg);
                    }
                }
            }
            finally
            {
                builder.Dispose();
            }
        }
Exemple #18
0
        private IEnumerable <Tombstone> ReadTombstones(INewDocumentActions actions = null)
        {
            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.StartArray)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            var context = _context;
            var builder = CreateBuilder(context, null);

            try
            {
                while (true)
                {
                    if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
                    }

                    if (_state.CurrentTokenType == JsonParserToken.EndArray)
                    {
                        break;
                    }

                    if (actions != null)
                    {
                        var oldContext = context;
                        context = actions.GetContextForNewDocument();
                        if (oldContext != context)
                        {
                            builder.Dispose();
                            builder = CreateBuilder(context, null);
                        }
                    }
                    builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);

                    _context.CachedProperties.NewDocument();

                    ReadObject(builder);

                    var data = builder.CreateReader();
                    builder.Reset();

                    var tombstone = new Tombstone();
                    if (data.TryGet("Key", out tombstone.LowerId) &&
                        data.TryGet(nameof(Tombstone.Type), out string type) &&
                        data.TryGet(nameof(Tombstone.Collection), out tombstone.Collection) &&
                        data.TryGet(nameof(Tombstone.LastModified), out tombstone.LastModified))
                    {
                        if (Enum.TryParse <Tombstone.TombstoneType>(type, out var tombstoneType) == false)
                        {
                            var msg = $"Ignoring a tombstone of type `{type}` which is not supported in 4.0. ";
                            if (_log.IsOperationsEnabled)
                            {
                                _log.Operations(msg);
                            }

                            _result.Tombstones.ErroredCount++;
                            _result.AddWarning(msg);
                            continue;
                        }

                        tombstone.Type = tombstoneType;
                        yield return(tombstone);
                    }
                    else
                    {
                        var msg = "Ignoring an invalid tombstone which you try to import. " + data;
                        if (_log.IsOperationsEnabled)
                        {
                            _log.Operations(msg);
                        }

                        _result.Tombstones.ErroredCount++;
                        _result.AddWarning(msg);
                    }
                }
            }
            finally
            {
                builder.Dispose();
            }
        }
Exemple #19
0
        private IEnumerable <DocumentItem> ReadDocuments(INewDocumentActions actions = null)
        {
            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.StartArray)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            var context      = _context;
            var legacyImport = _buildVersionType == BuildVersionType.V3;
            var modifier     = new BlittableMetadataModifier(context)
            {
                ReadFirstEtagOfLegacyRevision = legacyImport,
                ReadLegacyEtag = _readLegacyEtag,
                OperateOnTypes = _operateOnTypes
            };
            var builder = CreateBuilder(context, modifier);

            try
            {
                List <DocumentItem.AttachmentStream> attachments = null;
                while (true)
                {
                    if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
                    }

                    if (_state.CurrentTokenType == JsonParserToken.EndArray)
                    {
                        break;
                    }

                    if (actions != null)
                    {
                        var oldContext = context;
                        context = actions.GetContextForNewDocument();
                        if (oldContext != context)
                        {
                            builder.Dispose();
                            builder = CreateBuilder(context, modifier);
                        }
                    }
                    builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);

                    _context.CachedProperties.NewDocument();

                    ReadObject(builder);

                    var data = builder.CreateReader();
                    builder.Reset();

                    if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
                        metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type))
                    {
                        if (type != DocumentItem.ExportDocumentType.Attachment)
                        {
                            var msg = $"Ignoring an item of type `{type}`. " + data;
                            if (_log.IsOperationsEnabled)
                            {
                                _log.Operations(msg);
                            }
                            _result.AddWarning(msg);
                            continue;
                        }

                        if (attachments == null)
                        {
                            attachments = new List <DocumentItem.AttachmentStream>();
                        }

                        var attachment = new DocumentItem.AttachmentStream
                        {
                            Stream = actions.GetTempStream()
                        };
                        ProcessAttachmentStream(context, data, ref attachment);
                        attachments.Add(attachment);
                        continue;
                    }

                    if (legacyImport)
                    {
                        if (modifier.Id.Contains(HiLoHandler.RavenHiloIdPrefix))
                        {
                            data.Modifications = new DynamicJsonValue
                            {
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection
                                }
                            };
                        }
                    }

                    if (data.Modifications != null)
                    {
                        data = context.ReadObject(data, modifier.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
                    }

                    _result.LegacyLastDocumentEtag = modifier.LegacyEtag;

                    yield return(new DocumentItem
                    {
                        Document = new Document
                        {
                            Data = data,
                            Id = modifier.Id,
                            ChangeVector = modifier.ChangeVector,
                            Flags = modifier.Flags,
                            NonPersistentFlags = modifier.NonPersistentFlags,
                            LastModified = modifier.LastModified ?? _database.Time.GetUtcNow(),
                        },
                        Attachments = attachments
                    });

                    attachments = null;
                }
            }
            finally
            {
                builder.Dispose();
            }
        }
Exemple #20
0
        private IEnumerable <DocumentItem> ReadLegacyAttachments(INewDocumentActions actions)
        {
            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.StartArray)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            var context  = _context;
            var modifier = new BlittableMetadataModifier(context);
            var builder  = CreateBuilder(context, modifier);

            try
            {
                while (true)
                {
                    if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading legacy attachments", _peepingTomStream, _parser);
                    }

                    if (_state.CurrentTokenType == JsonParserToken.EndArray)
                    {
                        break;
                    }

                    if (actions != null)
                    {
                        var oldContext = context;
                        context = actions.GetContextForNewDocument();
                        if (oldContext != context)
                        {
                            builder.Dispose();
                            builder = CreateBuilder(context, modifier);
                        }
                    }
                    builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);

                    _context.CachedProperties.NewDocument();

                    ReadObject(builder);

                    var data = builder.CreateReader();
                    builder.Reset();

                    var attachment = new DocumentItem.AttachmentStream
                    {
                        Stream = actions.GetTempStream()
                    };

                    var attachmentInfo = ProcessLegacyAttachment(context, data, ref attachment);
                    if (ShouldSkip(attachmentInfo))
                    {
                        continue;
                    }

                    var dummyDoc = new DocumentItem
                    {
                        Document = new Document
                        {
                            Data               = WriteDummyDocumentForAttachment(context, attachmentInfo),
                            Id                 = attachmentInfo.Id,
                            ChangeVector       = string.Empty,
                            Flags              = DocumentFlags.HasAttachments,
                            NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler,
                            LastModified       = _database.Time.GetUtcNow(),
                        },
                        Attachments = new List <DocumentItem.AttachmentStream>
                        {
                            attachment
                        }
                    };

                    yield return(dummyDoc);
                }
            }
            finally
            {
                builder.Dispose();
            }
        }
Exemple #21
0
        private IEnumerable <BlittableJsonReaderObject> ReadArray(INewDocumentActions actions = null)
        {
            if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
            }

            if (_state.CurrentTokenType != JsonParserToken.StartArray)
            {
                UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, got " + _state.CurrentTokenType, _peepingTomStream, _parser);
            }

            var builder = CreateBuilder(_context, null);

            try
            {
                while (true)
                {
                    if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
                    {
                        UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading array", _peepingTomStream, _parser);
                    }

                    if (_state.CurrentTokenType == JsonParserToken.EndArray)
                    {
                        break;
                    }
                    if (actions != null)
                    {
                        var oldContext = _context;
                        var context    = actions.GetContextForNewDocument();
                        if (_context != oldContext)
                        {
                            builder.Dispose();
                            builder = CreateBuilder(context, null);
                        }
                    }
                    builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);

                    _context.CachedProperties.NewDocument();

                    ReadObject(builder);

                    var data = builder.CreateReader();
                    builder.Reset();

                    if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
                        metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type) &&
                        type == DocumentItem.ExportDocumentType.Attachment)
                    {
                        // skip document attachments, documents with attachments are handled separately
                        SkipAttachmentStream(data);
                        continue;
                    }

                    yield return(data);
                }
            }
            finally
            {
                builder.Dispose();
            }
        }