public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion) { _result = result; _returnBuffer = _context.GetManagedBuffer(out _buffer); _state = new JsonParserState(); _parser = new UnmanagedJsonParser(_context, _state, "file"); if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartObject) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } buildVersion = ReadBuildVersion(); _buildVersionType = BuildVersion.Type(buildVersion); _readLegacyEtag = options.ReadLegacyEtag; return(new DisposableAction(() => { _parser.Dispose(); _returnBuffer.Dispose(); _returnWriteBuffer.Dispose(); })); }
public async ValueTask DisposeAsync() { if (_disposed) { return; } if (_done == false) { while (await MoveNextAsync().ConfigureAwait(false)) { // we need to consume the rest of the stream, before we can move next the outer enumerator } } if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType != JsonParserToken.EndObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } _disposed = true; }
private long ReadBuildVersion() { var type = ReadType(); if (type == null) { return(0); } if (type.Equals("BuildVersion", StringComparison.OrdinalIgnoreCase) == false) { _currentType = GetType(type); return(0); } if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.Integer) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected integer BuildVersion, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } return(_state.Long); }
public async Task <bool> MoveNextAsync() { if (_initialized == false) { await InitializeAsync().ConfigureAwait(false); } if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType != JsonParserToken.EndObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } return(false); } using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state)) { await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer).ConfigureAwait(false); Current = builder.CreateReader(); return(true); } }
public IEnumerator <BlittableJsonReaderObject> SetResult(StreamResult response) { if (response == null) { throw new InvalidOperationException("The index does not exists, failed to stream results"); } var state = new JsonParserState(); JsonOperationContext.ManagedPinnedBuffer buffer; using (response.Response) using (response.Stream) using (var parser = new UnmanagedJsonParser(_session.Context, state, "stream contents")) using (_session.Context.GetManagedBuffer(out buffer)) using (var peepingTomStream = new PeepingTomStream(response.Stream, _session.Context)) { if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } if (state.CurrentTokenType != JsonParserToken.StartObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } if (_isQueryStream) { HandleStreamQueryStats(_session.Context, response, parser, state, buffer, _statistics); } var property = UnmanagedJsonParserHelper.ReadString(_session.Context, peepingTomStream, parser, state, buffer); if (string.Equals(property, "Results") == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } foreach (var result in UnmanagedJsonParserHelper.ReadArrayToMemory(_session.Context, peepingTomStream, parser, state, buffer)) { yield return(result); } if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } if (state.CurrentTokenType != JsonParserToken.EndObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } } }
public async Task <bool> MoveNextAsync() { if (_initialized == false) { await InitializeAsync().ConfigureAwait(false); } if (_docsCountOnCachedRenewSession <= 16 * 1024) { if (_cachedItemsRenew) { _session.Context.CachedProperties = new CachedProperties(_session.Context); ++_docsCountOnCachedRenewSession; } } else { _session.Context.Renew(); _docsCountOnCachedRenewSession = 0; } if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType != JsonParserToken.EndObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } return(false); } using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state)) { if (_cachedItemsRenew == false) { _cachedItemsRenew = builder.NeedResetPropertiesCache(); } await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer).ConfigureAwait(false); Current = builder.CreateReader(); return(true); } }
public async Task InitializeAsync() { var property = UnmanagedJsonParserHelper.ReadString(_context, _peepingTomStream, _parser, _state, _buffer); if (property.StartsWith(Constants.TimeSeries.QueryFunction) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType != JsonParserToken.StartArray) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); }
public async Task InitializeAsync() { AssertNotSync(); try { _initialized = true; _state = new JsonParserState(); _parser = new UnmanagedJsonParser(_session.Context, _state, "stream contents"); _returnBuffer = _session.Context.GetManagedBuffer(out _buffer); if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType != JsonParserToken.StartObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_isQueryStream) { HandleStreamQueryStats(_session.Context, _response, _parser, _state, _buffer, _streamQueryStatistics); } var property = UnmanagedJsonParserHelper.ReadString(_session.Context, _peepingTomStream, _parser, _state, _buffer); if (string.Equals(property, "Results") == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } } catch { Dispose(); throw; } }
private static void HandleStreamQueryStats(JsonOperationContext context, StreamResult response, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, StreamQueryStatistics streamQueryStatistics = null) { using (var peepingTomStream = new PeepingTomStream(response.Stream, context)) { var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.ResultEtag)) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); var resultEtag = UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer); property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.IsStale)) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); if (state.CurrentTokenType != JsonParserToken.False && state.CurrentTokenType != JsonParserToken.True) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); var isStale = state.CurrentTokenType != JsonParserToken.False; property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.IndexName)) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); var indexName = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.TotalResults)) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); var totalResults = (int)UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer); property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.IndexTimestamp)) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); var indexTimestamp = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (streamQueryStatistics == null) return; streamQueryStatistics.IndexName = indexName; streamQueryStatistics.IsStale = isStale; streamQueryStatistics.TotalResults = totalResults; streamQueryStatistics.ResultEtag = resultEtag; if (DateTime.TryParseExact(indexTimestamp, "o", CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out DateTime timeStamp) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); streamQueryStatistics.IndexTimestamp = timeStamp; } }
private unsafe string ReadType() { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of object when reading type", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndObject) { return(null); } if (_state.CurrentTokenType != JsonParserToken.String) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected property type to be string, but was " + _state.CurrentTokenType, _peepingTomStream, _parser); } return(_context.AllocateStringValue(null, _state.StringBuffer, _state.StringSize).ToString()); }
public async ValueTask<bool> MoveNextAsync() { if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType == JsonParserToken.EndArray) { _done = true; _current = null; return false; } using (var builder = new BlittableJsonDocumentBuilder(_context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", _parser, _state)) { await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false); _current = builder.CreateReader(); return true; } }
public bool MoveNext() { AssertInitialized(); CheckIfContextOrCacheNeedToBeRenewed(); _timeSeriesIt?.Dispose(); if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType == JsonParserToken.EndArray) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType != JsonParserToken.EndObject) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); return false; } _builder.Renew("readArray/singleResult", BlittableJsonDocumentBuilder.UsageMode.ToDisk); if (_isTimeSeriesStream) UnmanagedJsonParserHelper.ReadProperty(_builder, _peepingTomStream, _parser, _buffer); else UnmanagedJsonParserHelper.ReadObject(_builder, _peepingTomStream, _parser, _buffer); Current = _builder.CreateReader(); _builder.Reset(); if (_isTimeSeriesStream) { _timeSeriesIt = new TimeSeriesStreamEnumerator(_builderContext, _peepingTomStream, _parser, _state, _buffer); _timeSeriesIt.Initialize(); } return true; }
public async ValueTask<bool> MoveNextAsync() { AssertInitialized(); CheckIfContextOrCacheNeedToBeRenewed(); if (_timeSeriesIt != null) await _timeSeriesIt.DisposeAsync().ConfigureAwait(false); if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType == JsonParserToken.EndArray) { if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType != JsonParserToken.EndObject) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); return false; } _builder.Renew("readArray/singleResult", BlittableJsonDocumentBuilder.UsageMode.ToDisk); if (_isTimeSeriesStream) await UnmanagedJsonParserHelper.ReadPropertyAsync(_builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false); else await UnmanagedJsonParserHelper.ReadObjectAsync(_builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false); Current = _builder.CreateReader(); _builder.Reset(); if (_isTimeSeriesStream) { _timeSeriesIt = new TimeSeriesStreamEnumerator(_builderContext, _peepingTomStream, _parser, _state, _buffer); await _timeSeriesIt.InitializeAsync().ConfigureAwait(false); } return true; }
public async Task InitializeAsync() { AssertNotSync(); try { _initialized = true; _state = new JsonParserState(); _parser = new UnmanagedJsonParser(_session.Context, _state, "stream contents"); _builder = new BlittableJsonDocumentBuilder(_builderContext, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state); _returnBuffer = _session.Context.GetMemoryBuffer(out _buffer); if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType != JsonParserToken.StartObject) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_isQueryStream) HandleStreamQueryStats(_builderContext, _response, _parser, _state, _buffer, _streamQueryStatistics); var property = UnmanagedJsonParserHelper.ReadString(_builderContext, _peepingTomStream, _parser, _state, _buffer); if (string.Equals(property, "Results") == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); if (_state.CurrentTokenType != JsonParserToken.StartArray) UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } catch { await DisposeAsync().ConfigureAwait(false); throw; } }
public bool MoveNext() { AssertInitialized(); CheckIfContextNeedsToBeRenewed(); if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } if (_state.CurrentTokenType != JsonParserToken.EndObject) { UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream); } return(false); } using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state)) { if (_cachedItemsRenew == false) { _cachedItemsRenew = builder.NeedResetPropertiesCache(); } UnmanagedJsonParserHelper.ReadObject(builder, _peepingTomStream, _parser, _buffer); Current = builder.CreateReader(); return(true); } }
private void ReadObject(Action <BlittableJsonReaderObject> readAction) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartObject) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, got " + _state.CurrentTokenType, _peepingTomStream, _parser); } using (var builder = CreateBuilder(_context, null)) { _context.CachedProperties.NewDocument(); ReadObject(builder); using (var reader = builder.CreateReader()) { readAction(reader); } } }
private IEnumerable <DocumentConflict> ReadConflicts(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var builder = CreateBuilder(context, null); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, null); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); var conflict = new DocumentConflict(); if (data.TryGet(nameof(DocumentConflict.Id), out conflict.Id) && data.TryGet(nameof(DocumentConflict.Collection), out conflict.Collection) && data.TryGet(nameof(DocumentConflict.Flags), out string flags) && data.TryGet(nameof(DocumentConflict.ChangeVector), out conflict.ChangeVector) && data.TryGet(nameof(DocumentConflict.Etag), out conflict.Etag) && data.TryGet(nameof(DocumentConflict.LastModified), out conflict.LastModified) && data.TryGet(nameof(DocumentConflict.Doc), out conflict.Doc)) { conflict.Flags = Enum.Parse <DocumentFlags>(flags); if (conflict.Doc != null) // This is null for conflict that was generated from tombstone { conflict.Doc = context.ReadObject(conflict.Doc, conflict.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); } yield return(conflict); } else { var msg = "Ignoring an invalid conflict which you try to import. " + data; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.Conflicts.ErroredCount++; _result.AddWarning(msg); } } } finally { builder.Dispose(); } }
private IEnumerable <Tombstone> ReadTombstones(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var builder = CreateBuilder(context, null); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, null); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); var tombstone = new Tombstone(); if (data.TryGet("Key", out tombstone.LowerId) && data.TryGet(nameof(Tombstone.Type), out string type) && data.TryGet(nameof(Tombstone.Collection), out tombstone.Collection) && data.TryGet(nameof(Tombstone.LastModified), out tombstone.LastModified)) { if (Enum.TryParse <Tombstone.TombstoneType>(type, out var tombstoneType) == false) { var msg = $"Ignoring a tombstone of type `{type}` which is not supported in 4.0. "; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.Tombstones.ErroredCount++; _result.AddWarning(msg); continue; } tombstone.Type = tombstoneType; yield return(tombstone); } else { var msg = "Ignoring an invalid tombstone which you try to import. " + data; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.Tombstones.ErroredCount++; _result.AddWarning(msg); } } } finally { builder.Dispose(); } }
private IEnumerable <DocumentItem> ReadDocuments(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var legacyImport = _buildVersionType == BuildVersionType.V3; var modifier = new BlittableMetadataModifier(context) { ReadFirstEtagOfLegacyRevision = legacyImport, ReadLegacyEtag = _readLegacyEtag, OperateOnTypes = _operateOnTypes }; var builder = CreateBuilder(context, modifier); try { List <DocumentItem.AttachmentStream> attachments = null; while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, modifier); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) && metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type)) { if (type != DocumentItem.ExportDocumentType.Attachment) { var msg = $"Ignoring an item of type `{type}`. " + data; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.AddWarning(msg); continue; } if (attachments == null) { attachments = new List <DocumentItem.AttachmentStream>(); } var attachment = new DocumentItem.AttachmentStream { Stream = actions.GetTempStream() }; ProcessAttachmentStream(context, data, ref attachment); attachments.Add(attachment); continue; } if (legacyImport) { if (modifier.Id.Contains(HiLoHandler.RavenHiloIdPrefix)) { data.Modifications = new DynamicJsonValue { [Constants.Documents.Metadata.Key] = new DynamicJsonValue { [Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection } }; } } if (data.Modifications != null) { data = context.ReadObject(data, modifier.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); } _result.LegacyLastDocumentEtag = modifier.LegacyEtag; yield return(new DocumentItem { Document = new Document { Data = data, Id = modifier.Id, ChangeVector = modifier.ChangeVector, Flags = modifier.Flags, NonPersistentFlags = modifier.NonPersistentFlags, LastModified = modifier.LastModified ?? _database.Time.GetUtcNow(), }, Attachments = attachments }); attachments = null; } } finally { builder.Dispose(); } }
private IEnumerable <DocumentItem> ReadLegacyAttachments(INewDocumentActions actions) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var modifier = new BlittableMetadataModifier(context); var builder = CreateBuilder(context, modifier); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading legacy attachments", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, modifier); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); var attachment = new DocumentItem.AttachmentStream { Stream = actions.GetTempStream() }; var attachmentInfo = ProcessLegacyAttachment(context, data, ref attachment); if (ShouldSkip(attachmentInfo)) { continue; } var dummyDoc = new DocumentItem { Document = new Document { Data = WriteDummyDocumentForAttachment(context, attachmentInfo), Id = attachmentInfo.Id, ChangeVector = string.Empty, Flags = DocumentFlags.HasAttachments, NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler, LastModified = _database.Time.GetUtcNow(), }, Attachments = new List <DocumentItem.AttachmentStream> { attachment } }; yield return(dummyDoc); } } finally { builder.Dispose(); } }
private IEnumerable <BlittableJsonReaderObject> ReadArray(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var builder = CreateBuilder(_context, null); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading array", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = _context; var context = actions.GetContextForNewDocument(); if (_context != oldContext) { builder.Dispose(); builder = CreateBuilder(context, null); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) && metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type) && type == DocumentItem.ExportDocumentType.Attachment) { // skip document attachments, documents with attachments are handled separately SkipAttachmentStream(data); continue; } yield return(data); } } finally { builder.Dispose(); } }