private IEnumerable <DocumentItem> ReadDocuments(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var legacyImport = _buildVersionType == BuildVersionType.V3;
var modifier = new BlittableMetadataModifier(context)
{
ReadFirstEtagOfLegacyRevision = legacyImport,
ReadLegacyEtag = _readLegacyEtag,
OperateOnTypes = _operateOnTypes
};
var builder = CreateBuilder(context, modifier);
try
{
List <DocumentItem.AttachmentStream> attachments = null;
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, modifier);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type))
{
if (type != DocumentItem.ExportDocumentType.Attachment)
{
var msg = $"Ignoring an item of type `{type}`. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.AddWarning(msg);
continue;
}
if (attachments == null)
{
attachments = new List <DocumentItem.AttachmentStream>();
}
var attachment = new DocumentItem.AttachmentStream
{
Stream = actions.GetTempStream()
};
ProcessAttachmentStream(context, data, ref attachment);
attachments.Add(attachment);
continue;
}
if (legacyImport)
{
if (modifier.Id.Contains(HiLoHandler.RavenHiloIdPrefix))
{
data.Modifications = new DynamicJsonValue
{
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection
}
};
}
}
if (data.Modifications != null)
{
data = context.ReadObject(data, modifier.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
_result.LegacyLastDocumentEtag = modifier.LegacyEtag;
yield return(new DocumentItem
{
Document = new Document
{
Data = data,
Id = modifier.Id,
ChangeVector = modifier.ChangeVector,
Flags = modifier.Flags,
NonPersistentFlags = modifier.NonPersistentFlags,
LastModified = modifier.LastModified ?? _database.Time.GetUtcNow(),
},
Attachments = attachments
});
attachments = null;
}
}
finally
{
builder.Dispose();
}
}
private IEnumerable <DocumentItem> ReadLegacyAttachments(INewDocumentActions actions)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var modifier = new BlittableMetadataModifier(context);
var builder = CreateBuilder(context, modifier);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading legacy attachments", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, modifier);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var attachment = new DocumentItem.AttachmentStream
{
Stream = actions.GetTempStream()
};
var attachmentInfo = ProcessLegacyAttachment(context, data, ref attachment);
if (ShouldSkip(attachmentInfo))
{
continue;
}
var dummyDoc = new DocumentItem
{
Document = new Document
{
Data = WriteDummyDocumentForAttachment(context, attachmentInfo),
Id = attachmentInfo.Id,
ChangeVector = string.Empty,
Flags = DocumentFlags.HasAttachments,
NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler,
LastModified = _database.Time.GetUtcNow(),
},
Attachments = new List <DocumentItem.AttachmentStream>
{
attachment
}
};
yield return(dummyDoc);
}
}
finally
{
builder.Dispose();
}
}