private IEnumerable <DocumentItem> ReadDocuments(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var legacyImport = _buildVersionType == BuildVersionType.V3;
var modifier = new BlittableMetadataModifier(context)
{
ReadFirstEtagOfLegacyRevision = legacyImport,
ReadLegacyEtag = _readLegacyEtag,
OperateOnTypes = _operateOnTypes
};
var builder = CreateBuilder(context, modifier);
try
{
List <DocumentItem.AttachmentStream> attachments = null;
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, modifier);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type))
{
if (type != DocumentItem.ExportDocumentType.Attachment)
{
var msg = $"Ignoring an item of type `{type}`. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.AddWarning(msg);
continue;
}
if (attachments == null)
{
attachments = new List <DocumentItem.AttachmentStream>();
}
var attachment = new DocumentItem.AttachmentStream
{
Stream = actions.GetTempStream()
};
ProcessAttachmentStream(context, data, ref attachment);
attachments.Add(attachment);
continue;
}
if (legacyImport)
{
if (modifier.Id.Contains(HiLoHandler.RavenHiloIdPrefix))
{
data.Modifications = new DynamicJsonValue
{
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection
}
};
}
}
if (data.Modifications != null)
{
data = context.ReadObject(data, modifier.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
_result.LegacyLastDocumentEtag = modifier.LegacyEtag;
yield return(new DocumentItem
{
Document = new Document
{
Data = data,
Id = modifier.Id,
ChangeVector = modifier.ChangeVector,
Flags = modifier.Flags,
NonPersistentFlags = modifier.NonPersistentFlags,
LastModified = modifier.LastModified ?? _database.Time.GetUtcNow(),
},
Attachments = attachments
});
attachments = null;
}
}
finally
{
builder.Dispose();
}
}
private IEnumerable <BlittableJsonReaderObject> ReadArray(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var builder = CreateBuilder(_context, null);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading array", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = _context;
var context = actions.GetContextForNewDocument();
if (_context != oldContext)
{
builder.Dispose();
builder = CreateBuilder(context, null);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type) &&
type == DocumentItem.ExportDocumentType.Attachment)
{
// skip document attachments, documents with attachments are handled separately
SkipAttachmentStream(data);
continue;
}
yield return(data);
}
}
finally
{
builder.Dispose();
}
}
private IEnumerable <DocumentItem> ReadLegacyAttachments(INewDocumentActions actions)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var modifier = new BlittableMetadataModifier(context);
var builder = CreateBuilder(context, modifier);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading legacy attachments", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, modifier);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var attachment = new DocumentItem.AttachmentStream
{
Stream = actions.GetTempStream()
};
var attachmentInfo = ProcessLegacyAttachment(context, data, ref attachment);
if (ShouldSkip(attachmentInfo))
{
continue;
}
var dummyDoc = new DocumentItem
{
Document = new Document
{
Data = WriteDummyDocumentForAttachment(context, attachmentInfo),
Id = attachmentInfo.Id,
ChangeVector = string.Empty,
Flags = DocumentFlags.HasAttachments,
NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler,
LastModified = _database.Time.GetUtcNow(),
},
Attachments = new List <DocumentItem.AttachmentStream>
{
attachment
}
};
yield return(dummyDoc);
}
}
finally
{
builder.Dispose();
}
}
private IEnumerable <DocumentConflict> ReadConflicts(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var builder = CreateBuilder(context, null);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, null);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var conflict = new DocumentConflict();
if (data.TryGet(nameof(DocumentConflict.Id), out conflict.Id) &&
data.TryGet(nameof(DocumentConflict.Collection), out conflict.Collection) &&
data.TryGet(nameof(DocumentConflict.Flags), out string flags) &&
data.TryGet(nameof(DocumentConflict.ChangeVector), out conflict.ChangeVector) &&
data.TryGet(nameof(DocumentConflict.Etag), out conflict.Etag) &&
data.TryGet(nameof(DocumentConflict.LastModified), out conflict.LastModified) &&
data.TryGet(nameof(DocumentConflict.Doc), out conflict.Doc))
{
conflict.Flags = Enum.Parse <DocumentFlags>(flags);
if (conflict.Doc != null) // This is null for conflict that was generated from tombstone
{
conflict.Doc = context.ReadObject(conflict.Doc, conflict.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
yield return(conflict);
}
else
{
var msg = "Ignoring an invalid conflict which you try to import. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.Conflicts.ErroredCount++;
_result.AddWarning(msg);
}
}
}
finally
{
builder.Dispose();
}
}
private IEnumerable <Tombstone> ReadTombstones(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var builder = CreateBuilder(context, null);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, null);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var tombstone = new Tombstone();
if (data.TryGet("Key", out tombstone.LowerId) &&
data.TryGet(nameof(Tombstone.Type), out string type) &&
data.TryGet(nameof(Tombstone.Collection), out tombstone.Collection) &&
data.TryGet(nameof(Tombstone.LastModified), out tombstone.LastModified))
{
if (Enum.TryParse <Tombstone.TombstoneType>(type, out var tombstoneType) == false)
{
var msg = $"Ignoring a tombstone of type `{type}` which is not supported in 4.0. ";
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.Tombstones.ErroredCount++;
_result.AddWarning(msg);
continue;
}
tombstone.Type = tombstoneType;
yield return(tombstone);
}
else
{
var msg = "Ignoring an invalid tombstone which you try to import. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.Tombstones.ErroredCount++;
_result.AddWarning(msg);
}
}
}
finally
{
builder.Dispose();
}
}
private static void HandleStreamQueryStats(JsonOperationContext context, StreamResult response, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer, StreamQueryStatistics streamQueryStatistics = null)
{
using (var peepingTomStream = new PeepingTomStream(response.Stream, context))
{
var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.ResultEtag)) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
var resultEtag = UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer);
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.IsStale)) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.False && state.CurrentTokenType != JsonParserToken.True)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
var isStale = state.CurrentTokenType != JsonParserToken.False;
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.IndexName)) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
var indexName = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.TotalResults)) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
var totalResults = (int)UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer);
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.IndexTimestamp)) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
var indexTimestamp = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (streamQueryStatistics == null)
{
return;
}
streamQueryStatistics.IndexName = indexName;
streamQueryStatistics.IsStale = isStale;
streamQueryStatistics.TotalResults = totalResults;
streamQueryStatistics.ResultEtag = resultEtag;
if (DateTime.TryParseExact(indexTimestamp, "o", CultureInfo.InvariantCulture,
DateTimeStyles.RoundtripKind, out DateTime timeStamp) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
streamQueryStatistics.IndexTimestamp = timeStamp;
}
}
private static unsafe GetResponse ReadResponse(JsonOperationContext context, Stream stream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer)
{
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidResponse();
}
var getResponse = new GetResponse();
while (true)
{
if (UnmanagedJsonParserHelper.Read(stream, parser, state, buffer) == false)
{
ThrowInvalidResponse();
}
if (state.CurrentTokenType == JsonParserToken.EndObject)
{
break;
}
if (state.CurrentTokenType != JsonParserToken.String)
{
ThrowInvalidResponse();
}
var property = context.AllocateStringValue(null, state.StringBuffer, state.StringSize).ToString();
switch (property)
{
case nameof(GetResponse.Result):
if (UnmanagedJsonParserHelper.Read(stream, parser, state, buffer) == false)
{
ThrowInvalidResponse();
}
if (state.CurrentTokenType == JsonParserToken.Null)
{
continue;
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidResponse();
}
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "multi_get/result", parser, state))
{
UnmanagedJsonParserHelper.ReadObject(builder, stream, parser, buffer);
getResponse.Result = builder.CreateReader();
}
continue;
case nameof(GetResponse.Headers):
if (UnmanagedJsonParserHelper.Read(stream, parser, state, buffer) == false)
{
ThrowInvalidResponse();
}
if (state.CurrentTokenType == JsonParserToken.Null)
{
continue;
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidResponse();
}
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "multi_get/result", parser, state))
{
UnmanagedJsonParserHelper.ReadObject(builder, stream, parser, buffer);
using (var headersJson = builder.CreateReader())
{
foreach (var propertyName in headersJson.GetPropertyNames())
{
getResponse.Headers[propertyName] = headersJson[propertyName].ToString();
}
}
}
continue;
case nameof(GetResponse.StatusCode):
if (UnmanagedJsonParserHelper.Read(stream, parser, state, buffer) == false)
{
ThrowInvalidResponse();
}
if (state.CurrentTokenType != JsonParserToken.Integer)
{
ThrowInvalidResponse();
}
getResponse.StatusCode = (HttpStatusCode)state.Long;
continue;
default:
ThrowInvalidResponse();
break;
}
}
return(getResponse);
}
public void JsonDeserialize_WhenHasBlittableObjectPropertyAndWriteAndReadFromStream_ShouldResultInCommandWithTheProperty()
{
using (Server.ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var jsonSerializer = new JsonSerializer
{
ContractResolver = new DefaultRavenContractResolver(),
};
jsonSerializer.Converters.Add(BlittableJsonConverter.Instance);
var data = new { SomeProperty = "SomeValue" };
var expected = EntityToBlittable.ConvertCommandToBlittable(data, context);
var command = new Command {
BlittableObject = expected
};
//Serialize
BlittableJsonReaderObject toStream;
using (var writer = new BlittableJsonWriter(context))
{
jsonSerializer.Serialize(writer, command);
writer.FinalizeDocument();
toStream = writer.CreateReader();
}
//Simulates copying to file and loading
BlittableJsonReaderObject fromStream;
using (Stream stream = new MemoryStream())
{
//Pass to stream
using (var textWriter = new BlittableJsonTextWriter(context, stream))
{
context.Write(textWriter, toStream);
}
//Get from stream
stream.Position = 0;
var state = new JsonParserState();
var parser = new UnmanagedJsonParser(context, state, "some tag");
var peepingTomStream = new PeepingTomStream(stream, context);
using (context.GetManagedBuffer(out var buffer))
using (var builder =
new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "some tag", parser, state))
{
UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer);
UnmanagedJsonParserHelper.ReadObject(builder, peepingTomStream, parser, buffer);
fromStream = builder.CreateReader();
}
}
//Deserialize
BlittableJsonReaderObject actual;
using (var reader = new BlittableJsonReader(context))
{
reader.Init(fromStream);
var deserialized = jsonSerializer.Deserialize <Command>(reader);
actual = deserialized.BlittableObject;
}
Assert.Equal(expected, actual);
}
}
