public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_result = result;
_returnBuffer = _context.GetManagedBuffer(out _buffer);
_state = new JsonParserState();
_parser = new UnmanagedJsonParser(_context, _state, "file");
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
buildVersion = ReadBuildVersion();
_buildVersionType = BuildVersion.Type(buildVersion);
_readLegacyEtag = options.ReadLegacyEtag;
return(new DisposableAction(() =>
{
_parser.Dispose();
_returnBuffer.Dispose();
_returnWriteBuffer.Dispose();
}));
}
public async ValueTask DisposeAsync()
{
if (_disposed)
{
return;
}
if (_done == false)
{
while (await MoveNextAsync().ConfigureAwait(false))
{
// we need to consume the rest of the stream, before we can move next the outer enumerator
}
}
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType != JsonParserToken.EndObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
_disposed = true;
}
private long ReadBuildVersion()
{
var type = ReadType();
if (type == null)
{
return(0);
}
if (type.Equals("BuildVersion", StringComparison.OrdinalIgnoreCase) == false)
{
_currentType = GetType(type);
return(0);
}
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.Integer)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected integer BuildVersion, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
return(_state.Long);
}
public async Task <bool> MoveNextAsync()
{
if (_initialized == false)
{
await InitializeAsync().ConfigureAwait(false);
}
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType != JsonParserToken.EndObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
return(false);
}
using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state))
{
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer).ConfigureAwait(false);
Current = builder.CreateReader();
return(true);
}
}
public IEnumerator <BlittableJsonReaderObject> SetResult(StreamResult response)
{
if (response == null)
{
throw new InvalidOperationException("The index does not exists, failed to stream results");
}
var state = new JsonParserState();
JsonOperationContext.ManagedPinnedBuffer buffer;
using (response.Response)
using (response.Stream)
using (var parser = new UnmanagedJsonParser(_session.Context, state, "stream contents"))
using (_session.Context.GetManagedBuffer(out buffer))
using (var peepingTomStream = new PeepingTomStream(response.Stream, _session.Context))
{
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
if (_isQueryStream)
{
HandleStreamQueryStats(_session.Context, response, parser, state, buffer, _statistics);
}
var property = UnmanagedJsonParserHelper.ReadString(_session.Context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, "Results") == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
foreach (var result in UnmanagedJsonParserHelper.ReadArrayToMemory(_session.Context, peepingTomStream, parser, state, buffer))
{
yield return(result);
}
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
}
}
public async Task <bool> MoveNextAsync()
{
if (_initialized == false)
{
await InitializeAsync().ConfigureAwait(false);
}
if (_docsCountOnCachedRenewSession <= 16 * 1024)
{
if (_cachedItemsRenew)
{
_session.Context.CachedProperties = new CachedProperties(_session.Context);
++_docsCountOnCachedRenewSession;
}
}
else
{
_session.Context.Renew();
_docsCountOnCachedRenewSession = 0;
}
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType != JsonParserToken.EndObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
return(false);
}
using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state))
{
if (_cachedItemsRenew == false)
{
_cachedItemsRenew = builder.NeedResetPropertiesCache();
}
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer).ConfigureAwait(false);
Current = builder.CreateReader();
return(true);
}
}
public async Task InitializeAsync()
{
var property = UnmanagedJsonParserHelper.ReadString(_context, _peepingTomStream, _parser, _state, _buffer);
if (property.StartsWith(Constants.TimeSeries.QueryFunction) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType != JsonParserToken.StartArray)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
public async Task InitializeAsync()
{
AssertNotSync();
try
{
_initialized = true;
_state = new JsonParserState();
_parser = new UnmanagedJsonParser(_session.Context, _state, "stream contents");
_returnBuffer = _session.Context.GetManagedBuffer(out _buffer);
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType != JsonParserToken.StartObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_isQueryStream)
{
HandleStreamQueryStats(_session.Context, _response, _parser, _state, _buffer, _streamQueryStatistics);
}
var property = UnmanagedJsonParserHelper.ReadString(_session.Context, _peepingTomStream, _parser, _state, _buffer);
if (string.Equals(property, "Results") == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
}
catch
{
Dispose();
throw;
}
}
private static void HandleStreamQueryStats(JsonOperationContext context, StreamResult response, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, StreamQueryStatistics streamQueryStatistics = null)
{
using (var peepingTomStream = new PeepingTomStream(response.Stream, context))
{
var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.ResultEtag)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var resultEtag = UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer);
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.IsStale)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
if (state.CurrentTokenType != JsonParserToken.False && state.CurrentTokenType != JsonParserToken.True)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var isStale = state.CurrentTokenType != JsonParserToken.False;
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.IndexName)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var indexName = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.TotalResults)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var totalResults = (int)UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer);
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.IndexTimestamp)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var indexTimestamp = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (streamQueryStatistics == null)
return;
streamQueryStatistics.IndexName = indexName;
streamQueryStatistics.IsStale = isStale;
streamQueryStatistics.TotalResults = totalResults;
streamQueryStatistics.ResultEtag = resultEtag;
if (DateTime.TryParseExact(indexTimestamp, "o", CultureInfo.InvariantCulture,
DateTimeStyles.RoundtripKind, out DateTime timeStamp) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
streamQueryStatistics.IndexTimestamp = timeStamp;
}
}
private unsafe string ReadType()
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of object when reading type", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndObject)
{
return(null);
}
if (_state.CurrentTokenType != JsonParserToken.String)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected property type to be string, but was " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
return(_context.AllocateStringValue(null, _state.StringBuffer, _state.StringSize).ToString());
}
public async ValueTask<bool> MoveNextAsync()
{
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
_done = true;
_current = null;
return false;
}
using (var builder = new BlittableJsonDocumentBuilder(_context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", _parser, _state))
{
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false);
_current = builder.CreateReader();
return true;
}
}
public bool MoveNext()
{
AssertInitialized();
CheckIfContextOrCacheNeedToBeRenewed();
_timeSeriesIt?.Dispose();
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType != JsonParserToken.EndObject)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
return false;
}
_builder.Renew("readArray/singleResult", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
if (_isTimeSeriesStream)
UnmanagedJsonParserHelper.ReadProperty(_builder, _peepingTomStream, _parser, _buffer);
else
UnmanagedJsonParserHelper.ReadObject(_builder, _peepingTomStream, _parser, _buffer);
Current = _builder.CreateReader();
_builder.Reset();
if (_isTimeSeriesStream)
{
_timeSeriesIt = new TimeSeriesStreamEnumerator(_builderContext, _peepingTomStream, _parser, _state, _buffer);
_timeSeriesIt.Initialize();
}
return true;
}
public async ValueTask<bool> MoveNextAsync()
{
AssertInitialized();
CheckIfContextOrCacheNeedToBeRenewed();
if (_timeSeriesIt != null)
await _timeSeriesIt.DisposeAsync().ConfigureAwait(false);
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType != JsonParserToken.EndObject)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
return false;
}
_builder.Renew("readArray/singleResult", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
if (_isTimeSeriesStream)
await UnmanagedJsonParserHelper.ReadPropertyAsync(_builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false);
else
await UnmanagedJsonParserHelper.ReadObjectAsync(_builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false);
Current = _builder.CreateReader();
_builder.Reset();
if (_isTimeSeriesStream)
{
_timeSeriesIt = new TimeSeriesStreamEnumerator(_builderContext, _peepingTomStream, _parser, _state, _buffer);
await _timeSeriesIt.InitializeAsync().ConfigureAwait(false);
}
return true;
}
public async Task InitializeAsync()
{
AssertNotSync();
try
{
_initialized = true;
_state = new JsonParserState();
_parser = new UnmanagedJsonParser(_session.Context, _state, "stream contents");
_builder = new BlittableJsonDocumentBuilder(_builderContext, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state);
_returnBuffer = _session.Context.GetMemoryBuffer(out _buffer);
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType != JsonParserToken.StartObject)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_isQueryStream)
HandleStreamQueryStats(_builderContext, _response, _parser, _state, _buffer, _streamQueryStatistics);
var property = UnmanagedJsonParserHelper.ReadString(_builderContext, _peepingTomStream, _parser, _state, _buffer);
if (string.Equals(property, "Results") == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType != JsonParserToken.StartArray)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
catch
{
await DisposeAsync().ConfigureAwait(false);
throw;
}
}
public bool MoveNext()
{
AssertInitialized();
CheckIfContextNeedsToBeRenewed();
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType != JsonParserToken.EndObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
return(false);
}
using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state))
{
if (_cachedItemsRenew == false)
{
_cachedItemsRenew = builder.NeedResetPropertiesCache();
}
UnmanagedJsonParserHelper.ReadObject(builder, _peepingTomStream, _parser, _buffer);
Current = builder.CreateReader();
return(true);
}
}
private void ReadObject(Action <BlittableJsonReaderObject> readAction)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
using (var builder = CreateBuilder(_context, null))
{
_context.CachedProperties.NewDocument();
ReadObject(builder);
using (var reader = builder.CreateReader())
{
readAction(reader);
}
}
}
private IEnumerable <DocumentConflict> ReadConflicts(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var builder = CreateBuilder(context, null);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, null);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var conflict = new DocumentConflict();
if (data.TryGet(nameof(DocumentConflict.Id), out conflict.Id) &&
data.TryGet(nameof(DocumentConflict.Collection), out conflict.Collection) &&
data.TryGet(nameof(DocumentConflict.Flags), out string flags) &&
data.TryGet(nameof(DocumentConflict.ChangeVector), out conflict.ChangeVector) &&
data.TryGet(nameof(DocumentConflict.Etag), out conflict.Etag) &&
data.TryGet(nameof(DocumentConflict.LastModified), out conflict.LastModified) &&
data.TryGet(nameof(DocumentConflict.Doc), out conflict.Doc))
{
conflict.Flags = Enum.Parse <DocumentFlags>(flags);
if (conflict.Doc != null) // This is null for conflict that was generated from tombstone
{
conflict.Doc = context.ReadObject(conflict.Doc, conflict.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
yield return(conflict);
}
else
{
var msg = "Ignoring an invalid conflict which you try to import. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.Conflicts.ErroredCount++;
_result.AddWarning(msg);
}
}
}
finally
{
builder.Dispose();
}
}
private IEnumerable <Tombstone> ReadTombstones(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var builder = CreateBuilder(context, null);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, null);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var tombstone = new Tombstone();
if (data.TryGet("Key", out tombstone.LowerId) &&
data.TryGet(nameof(Tombstone.Type), out string type) &&
data.TryGet(nameof(Tombstone.Collection), out tombstone.Collection) &&
data.TryGet(nameof(Tombstone.LastModified), out tombstone.LastModified))
{
if (Enum.TryParse <Tombstone.TombstoneType>(type, out var tombstoneType) == false)
{
var msg = $"Ignoring a tombstone of type `{type}` which is not supported in 4.0. ";
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.Tombstones.ErroredCount++;
_result.AddWarning(msg);
continue;
}
tombstone.Type = tombstoneType;
yield return(tombstone);
}
else
{
var msg = "Ignoring an invalid tombstone which you try to import. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.Tombstones.ErroredCount++;
_result.AddWarning(msg);
}
}
}
finally
{
builder.Dispose();
}
}
private IEnumerable <DocumentItem> ReadDocuments(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var legacyImport = _buildVersionType == BuildVersionType.V3;
var modifier = new BlittableMetadataModifier(context)
{
ReadFirstEtagOfLegacyRevision = legacyImport,
ReadLegacyEtag = _readLegacyEtag,
OperateOnTypes = _operateOnTypes
};
var builder = CreateBuilder(context, modifier);
try
{
List <DocumentItem.AttachmentStream> attachments = null;
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, modifier);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type))
{
if (type != DocumentItem.ExportDocumentType.Attachment)
{
var msg = $"Ignoring an item of type `{type}`. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.AddWarning(msg);
continue;
}
if (attachments == null)
{
attachments = new List <DocumentItem.AttachmentStream>();
}
var attachment = new DocumentItem.AttachmentStream
{
Stream = actions.GetTempStream()
};
ProcessAttachmentStream(context, data, ref attachment);
attachments.Add(attachment);
continue;
}
if (legacyImport)
{
if (modifier.Id.Contains(HiLoHandler.RavenHiloIdPrefix))
{
data.Modifications = new DynamicJsonValue
{
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection
}
};
}
}
if (data.Modifications != null)
{
data = context.ReadObject(data, modifier.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
_result.LegacyLastDocumentEtag = modifier.LegacyEtag;
yield return(new DocumentItem
{
Document = new Document
{
Data = data,
Id = modifier.Id,
ChangeVector = modifier.ChangeVector,
Flags = modifier.Flags,
NonPersistentFlags = modifier.NonPersistentFlags,
LastModified = modifier.LastModified ?? _database.Time.GetUtcNow(),
},
Attachments = attachments
});
attachments = null;
}
}
finally
{
builder.Dispose();
}
}
private IEnumerable <DocumentItem> ReadLegacyAttachments(INewDocumentActions actions)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var modifier = new BlittableMetadataModifier(context);
var builder = CreateBuilder(context, modifier);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading legacy attachments", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, modifier);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var attachment = new DocumentItem.AttachmentStream
{
Stream = actions.GetTempStream()
};
var attachmentInfo = ProcessLegacyAttachment(context, data, ref attachment);
if (ShouldSkip(attachmentInfo))
{
continue;
}
var dummyDoc = new DocumentItem
{
Document = new Document
{
Data = WriteDummyDocumentForAttachment(context, attachmentInfo),
Id = attachmentInfo.Id,
ChangeVector = string.Empty,
Flags = DocumentFlags.HasAttachments,
NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler,
LastModified = _database.Time.GetUtcNow(),
},
Attachments = new List <DocumentItem.AttachmentStream>
{
attachment
}
};
yield return(dummyDoc);
}
}
finally
{
builder.Dispose();
}
}
private IEnumerable <BlittableJsonReaderObject> ReadArray(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var builder = CreateBuilder(_context, null);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading array", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = _context;
var context = actions.GetContextForNewDocument();
if (_context != oldContext)
{
builder.Dispose();
builder = CreateBuilder(context, null);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type) &&
type == DocumentItem.ExportDocumentType.Attachment)
{
// skip document attachments, documents with attachments are handled separately
SkipAttachmentStream(data);
continue;
}
yield return(data);
}
}
finally
{
builder.Dispose();
}
}