private IEnumerable <DocumentItem> ReadDocuments(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var legacyImport = _buildVersionType == BuildVersionType.V3; var modifier = new BlittableMetadataModifier(context) { ReadFirstEtagOfLegacyRevision = legacyImport, ReadLegacyEtag = _readLegacyEtag, OperateOnTypes = _operateOnTypes }; var builder = CreateBuilder(context, modifier); try { List <DocumentItem.AttachmentStream> attachments = null; while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, modifier); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) && metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type)) { if (type != DocumentItem.ExportDocumentType.Attachment) { var msg = $"Ignoring an item of type `{type}`. " + data; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.AddWarning(msg); continue; } if (attachments == null) { attachments = new List <DocumentItem.AttachmentStream>(); } var attachment = new DocumentItem.AttachmentStream { Stream = actions.GetTempStream() }; ProcessAttachmentStream(context, data, ref attachment); attachments.Add(attachment); continue; } if (legacyImport) { if (modifier.Id.Contains(HiLoHandler.RavenHiloIdPrefix)) { data.Modifications = new DynamicJsonValue { [Constants.Documents.Metadata.Key] = new DynamicJsonValue { [Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection } }; } } if (data.Modifications != null) { data = context.ReadObject(data, modifier.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); } _result.LegacyLastDocumentEtag = modifier.LegacyEtag; yield return(new DocumentItem { Document = new Document { Data = data, Id = modifier.Id, ChangeVector = modifier.ChangeVector, Flags = modifier.Flags, NonPersistentFlags = modifier.NonPersistentFlags, LastModified = modifier.LastModified ?? _database.Time.GetUtcNow(), }, Attachments = attachments }); attachments = null; } } finally { builder.Dispose(); } }
private IEnumerable <BlittableJsonReaderObject> ReadArray(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var builder = CreateBuilder(_context, null); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading array", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = _context; var context = actions.GetContextForNewDocument(); if (_context != oldContext) { builder.Dispose(); builder = CreateBuilder(context, null); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) && metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type) && type == DocumentItem.ExportDocumentType.Attachment) { // skip document attachments, documents with attachments are handled separately SkipAttachmentStream(data); continue; } yield return(data); } } finally { builder.Dispose(); } }
private IEnumerable <DocumentItem> ReadLegacyAttachments(INewDocumentActions actions) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var modifier = new BlittableMetadataModifier(context); var builder = CreateBuilder(context, modifier); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading legacy attachments", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, modifier); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); var attachment = new DocumentItem.AttachmentStream { Stream = actions.GetTempStream() }; var attachmentInfo = ProcessLegacyAttachment(context, data, ref attachment); if (ShouldSkip(attachmentInfo)) { continue; } var dummyDoc = new DocumentItem { Document = new Document { Data = WriteDummyDocumentForAttachment(context, attachmentInfo), Id = attachmentInfo.Id, ChangeVector = string.Empty, Flags = DocumentFlags.HasAttachments, NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler, LastModified = _database.Time.GetUtcNow(), }, Attachments = new List <DocumentItem.AttachmentStream> { attachment } }; yield return(dummyDoc); } } finally { builder.Dispose(); } }
private IEnumerable <DocumentConflict> ReadConflicts(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var builder = CreateBuilder(context, null); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, null); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); var conflict = new DocumentConflict(); if (data.TryGet(nameof(DocumentConflict.Id), out conflict.Id) && data.TryGet(nameof(DocumentConflict.Collection), out conflict.Collection) && data.TryGet(nameof(DocumentConflict.Flags), out string flags) && data.TryGet(nameof(DocumentConflict.ChangeVector), out conflict.ChangeVector) && data.TryGet(nameof(DocumentConflict.Etag), out conflict.Etag) && data.TryGet(nameof(DocumentConflict.LastModified), out conflict.LastModified) && data.TryGet(nameof(DocumentConflict.Doc), out conflict.Doc)) { conflict.Flags = Enum.Parse <DocumentFlags>(flags); if (conflict.Doc != null) // This is null for conflict that was generated from tombstone { conflict.Doc = context.ReadObject(conflict.Doc, conflict.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); } yield return(conflict); } else { var msg = "Ignoring an invalid conflict which you try to import. " + data; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.Conflicts.ErroredCount++; _result.AddWarning(msg); } } } finally { builder.Dispose(); } }
private IEnumerable <Tombstone> ReadTombstones(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var builder = CreateBuilder(context, null); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, null); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); var tombstone = new Tombstone(); if (data.TryGet("Key", out tombstone.LowerId) && data.TryGet(nameof(Tombstone.Type), out string type) && data.TryGet(nameof(Tombstone.Collection), out tombstone.Collection) && data.TryGet(nameof(Tombstone.LastModified), out tombstone.LastModified)) { if (Enum.TryParse <Tombstone.TombstoneType>(type, out var tombstoneType) == false) { var msg = $"Ignoring a tombstone of type `{type}` which is not supported in 4.0. "; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.Tombstones.ErroredCount++; _result.AddWarning(msg); continue; } tombstone.Type = tombstoneType; yield return(tombstone); } else { var msg = "Ignoring an invalid tombstone which you try to import. " + data; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.Tombstones.ErroredCount++; _result.AddWarning(msg); } } } finally { builder.Dispose(); } }
private static void HandleStreamQueryStats(JsonOperationContext context, StreamResult response, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer, StreamQueryStatistics streamQueryStatistics = null) { using (var peepingTomStream = new PeepingTomStream(response.Stream, context)) { var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.ResultEtag)) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } var resultEtag = UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer); property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.IsStale)) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } if (state.CurrentTokenType != JsonParserToken.False && state.CurrentTokenType != JsonParserToken.True) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } var isStale = state.CurrentTokenType != JsonParserToken.False; property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.IndexName)) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } var indexName = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.TotalResults)) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } var totalResults = (int)UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer); property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (string.Equals(property, nameof(StreamQueryStatistics.IndexTimestamp)) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } var indexTimestamp = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer); if (streamQueryStatistics == null) { return; } streamQueryStatistics.IndexName = indexName; streamQueryStatistics.IsStale = isStale; streamQueryStatistics.TotalResults = totalResults; streamQueryStatistics.ResultEtag = resultEtag; if (DateTime.TryParseExact(indexTimestamp, "o", CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out DateTime timeStamp) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream); } streamQueryStatistics.IndexTimestamp = timeStamp; } }
private static unsafe GetResponse ReadResponse(JsonOperationContext context, Stream stream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer) { if (state.CurrentTokenType != JsonParserToken.StartObject) { ThrowInvalidResponse(); } var getResponse = new GetResponse(); while (true) { if (UnmanagedJsonParserHelper.Read(stream, parser, state, buffer) == false) { ThrowInvalidResponse(); } if (state.CurrentTokenType == JsonParserToken.EndObject) { break; } if (state.CurrentTokenType != JsonParserToken.String) { ThrowInvalidResponse(); } var property = context.AllocateStringValue(null, state.StringBuffer, state.StringSize).ToString(); switch (property) { case nameof(GetResponse.Result): if (UnmanagedJsonParserHelper.Read(stream, parser, state, buffer) == false) { ThrowInvalidResponse(); } if (state.CurrentTokenType == JsonParserToken.Null) { continue; } if (state.CurrentTokenType != JsonParserToken.StartObject) { ThrowInvalidResponse(); } using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "multi_get/result", parser, state)) { UnmanagedJsonParserHelper.ReadObject(builder, stream, parser, buffer); getResponse.Result = builder.CreateReader(); } continue; case nameof(GetResponse.Headers): if (UnmanagedJsonParserHelper.Read(stream, parser, state, buffer) == false) { ThrowInvalidResponse(); } if (state.CurrentTokenType == JsonParserToken.Null) { continue; } if (state.CurrentTokenType != JsonParserToken.StartObject) { ThrowInvalidResponse(); } using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "multi_get/result", parser, state)) { UnmanagedJsonParserHelper.ReadObject(builder, stream, parser, buffer); using (var headersJson = builder.CreateReader()) { foreach (var propertyName in headersJson.GetPropertyNames()) { getResponse.Headers[propertyName] = headersJson[propertyName].ToString(); } } } continue; case nameof(GetResponse.StatusCode): if (UnmanagedJsonParserHelper.Read(stream, parser, state, buffer) == false) { ThrowInvalidResponse(); } if (state.CurrentTokenType != JsonParserToken.Integer) { ThrowInvalidResponse(); } getResponse.StatusCode = (HttpStatusCode)state.Long; continue; default: ThrowInvalidResponse(); break; } } return(getResponse); }
public void JsonDeserialize_WhenHasBlittableObjectPropertyAndWriteAndReadFromStream_ShouldResultInCommandWithTheProperty() { using (Server.ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { var jsonSerializer = new JsonSerializer { ContractResolver = new DefaultRavenContractResolver(), }; jsonSerializer.Converters.Add(BlittableJsonConverter.Instance); var data = new { SomeProperty = "SomeValue" }; var expected = EntityToBlittable.ConvertCommandToBlittable(data, context); var command = new Command { BlittableObject = expected }; //Serialize BlittableJsonReaderObject toStream; using (var writer = new BlittableJsonWriter(context)) { jsonSerializer.Serialize(writer, command); writer.FinalizeDocument(); toStream = writer.CreateReader(); } //Simulates copying to file and loading BlittableJsonReaderObject fromStream; using (Stream stream = new MemoryStream()) { //Pass to stream using (var textWriter = new BlittableJsonTextWriter(context, stream)) { context.Write(textWriter, toStream); } //Get from stream stream.Position = 0; var state = new JsonParserState(); var parser = new UnmanagedJsonParser(context, state, "some tag"); var peepingTomStream = new PeepingTomStream(stream, context); using (context.GetManagedBuffer(out var buffer)) using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "some tag", parser, state)) { UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer); UnmanagedJsonParserHelper.ReadObject(builder, peepingTomStream, parser, buffer); fromStream = builder.CreateReader(); } } //Deserialize BlittableJsonReaderObject actual; using (var reader = new BlittableJsonReader(context)) { reader.Init(fromStream); var deserialized = jsonSerializer.Deserialize <Command>(reader); actual = deserialized.BlittableObject; } Assert.Equal(expected, actual); } }