private long ReadBuildVersion()
{
var type = ReadType();
if (type == null)
{
return(0);
}
if (type.Equals("BuildVersion", StringComparison.OrdinalIgnoreCase) == false)
{
_currentType = GetType(type);
return(0);
}
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.Integer)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected integer BuildVersion, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
return(_state.Long);
}
public async ValueTask DisposeAsync()
{
if (_disposed)
{
return;
}
if (_done == false)
{
while (await MoveNextAsync().ConfigureAwait(false))
{
// we need to consume the rest of the stream, before we can move next the outer enumerator
}
}
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType != JsonParserToken.EndObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
_disposed = true;
}
private static IEnumerable <GetResponse> ReadResponses(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer)
{
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.StartArray)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
while (true)
{
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
yield return(ReadResponse(context, peepingTomStream, parser, state, buffer));
}
}
public static async Task <BlittableJsonReaderObject> SimulateSavingToFileAndLoadingAsync(JsonOperationContext context, BlittableJsonReaderObject toStream)
{
//Simulates saving to file and loading
BlittableJsonReaderObject fromStream;
await using (Stream stream = new MemoryStream())
{
//Pass to stream
await using (var textWriter = new AsyncBlittableJsonTextWriter(context, stream))
{
context.Write(textWriter, toStream);
}
//Get from stream
stream.Position = 0;
var state = new JsonParserState();
var parser = new UnmanagedJsonParser(context, state, "some tag");
var peepingTomStream = new PeepingTomStream(stream, context);
using (context.GetMemoryBuffer(out var buffer))
using (var builder =
new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "some tag", parser, state))
{
await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer);
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer);
fromStream = builder.CreateReader();
}
}
return(fromStream);
}
public async Task <bool> MoveNextAsync()
{
if (_initialized == false)
{
await InitializeAsync().ConfigureAwait(false);
}
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType != JsonParserToken.EndObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
return(false);
}
using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state))
{
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer).ConfigureAwait(false);
Current = builder.CreateReader();
return(true);
}
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_result = result;
_returnBuffer = _context.GetManagedBuffer(out _buffer);
_state = new JsonParserState();
_parser = new UnmanagedJsonParser(_context, _state, "file");
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
buildVersion = ReadBuildVersion();
_buildVersionType = BuildVersion.Type(buildVersion);
_readLegacyEtag = options.ReadLegacyEtag;
return(new DisposableAction(() =>
{
_parser.Dispose();
_returnBuffer.Dispose();
_returnWriteBuffer.Dispose();
}));
}
public IEnumerator <BlittableJsonReaderObject> SetResult(StreamResult response)
{
if (response == null)
{
throw new InvalidOperationException("The index does not exists, failed to stream results");
}
var state = new JsonParserState();
JsonOperationContext.ManagedPinnedBuffer buffer;
using (response.Response)
using (response.Stream)
using (var parser = new UnmanagedJsonParser(_session.Context, state, "stream contents"))
using (_session.Context.GetManagedBuffer(out buffer))
using (var peepingTomStream = new PeepingTomStream(response.Stream, _session.Context))
{
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
if (_isQueryStream)
{
HandleStreamQueryStats(_session.Context, response, parser, state, buffer, _statistics);
}
var property = UnmanagedJsonParserHelper.ReadString(_session.Context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, "Results") == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
foreach (var result in UnmanagedJsonParserHelper.ReadArrayToMemory(_session.Context, peepingTomStream, parser, state, buffer))
{
yield return(result);
}
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
}
}
public async Task <bool> MoveNextAsync()
{
if (_initialized == false)
{
await InitializeAsync().ConfigureAwait(false);
}
if (_docsCountOnCachedRenewSession <= 16 * 1024)
{
if (_cachedItemsRenew)
{
_session.Context.CachedProperties = new CachedProperties(_session.Context);
++_docsCountOnCachedRenewSession;
}
}
else
{
_session.Context.Renew();
_docsCountOnCachedRenewSession = 0;
}
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType != JsonParserToken.EndObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
return(false);
}
using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state))
{
if (_cachedItemsRenew == false)
{
_cachedItemsRenew = builder.NeedResetPropertiesCache();
}
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer).ConfigureAwait(false);
Current = builder.CreateReader();
return(true);
}
}
public override void SetResponseRaw(HttpResponseMessage response, Stream stream, JsonOperationContext context)
{
var state = new JsonParserState();
using (var parser = new UnmanagedJsonParser(context, state, "multi_get/response"))
using (context.GetMemoryBuffer(out var buffer))
using (var peepingTomStream = new PeepingTomStream(stream, context))
using (_cached)
{
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (property != nameof(BlittableArrayResult.Results))
{
ThrowInvalidJsonResponse(peepingTomStream);
}
var i = 0;
Result = new List <GetResponse>(_commands.Count);
foreach (var getResponse in ReadResponses(context, peepingTomStream, parser, state, buffer))
{
var command = _commands[i];
MaybeSetCache(getResponse, command);
Result.Add(_cached != null && getResponse.StatusCode == HttpStatusCode.NotModified
? new GetResponse {
Result = _cached.Values[i].Cached.Clone(context), StatusCode = HttpStatusCode.NotModified
}
: getResponse);
i++;
}
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
}
}
public async Task InitializeAsync()
{
var property = UnmanagedJsonParserHelper.ReadString(_context, _peepingTomStream, _parser, _state, _buffer);
if (property.StartsWith(Constants.TimeSeries.QueryFunction) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType != JsonParserToken.StartArray)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
public async Task InitializeAsync()
{
AssertNotSync();
try
{
_initialized = true;
_state = new JsonParserState();
_parser = new UnmanagedJsonParser(_session.Context, _state, "stream contents");
_returnBuffer = _session.Context.GetManagedBuffer(out _buffer);
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType != JsonParserToken.StartObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_isQueryStream)
{
HandleStreamQueryStats(_session.Context, _response, _parser, _state, _buffer, _streamQueryStatistics);
}
var property = UnmanagedJsonParserHelper.ReadString(_session.Context, _peepingTomStream, _parser, _state, _buffer);
if (string.Equals(property, "Results") == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer).ConfigureAwait(false) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
}
catch
{
Dispose();
throw;
}
}
public override void SetResponseRaw(HttpResponseMessage response, Stream stream, JsonOperationContext context)
{
var state = new JsonParserState();
using (var parser = new UnmanagedJsonParser(context, state, "multi_get/response"))
using (context.GetManagedBuffer(out JsonOperationContext.ManagedPinnedBuffer buffer))
using (var peepingTomStream = new PeepingTomStream(stream, context))
{
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (property != nameof(BlittableArrayResult.Results))
{
ThrowInvalidJsonResponse(peepingTomStream);
}
var i = 0;
Result = new List <GetResponse>();
foreach (var getResponse in ReadResponses(context, peepingTomStream, parser, state, buffer))
{
var command = _commands[i];
MaybeSetCache(getResponse, command);
MaybeReadFromCache(getResponse, command, context);
Result.Add(getResponse);
i++;
}
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
}
}
private static void HandleStreamQueryStats(JsonOperationContext context, StreamResult response, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, StreamQueryStatistics streamQueryStatistics = null)
{
using (var peepingTomStream = new PeepingTomStream(response.Stream, context))
{
var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.ResultEtag)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var resultEtag = UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer);
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.IsStale)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
if (state.CurrentTokenType != JsonParserToken.False && state.CurrentTokenType != JsonParserToken.True)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var isStale = state.CurrentTokenType != JsonParserToken.False;
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.IndexName)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var indexName = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.TotalResults)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var totalResults = (int)UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer);
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.IndexTimestamp)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var indexTimestamp = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (streamQueryStatistics == null)
return;
streamQueryStatistics.IndexName = indexName;
streamQueryStatistics.IsStale = isStale;
streamQueryStatistics.TotalResults = totalResults;
streamQueryStatistics.ResultEtag = resultEtag;
if (DateTime.TryParseExact(indexTimestamp, "o", CultureInfo.InvariantCulture,
DateTimeStyles.RoundtripKind, out DateTime timeStamp) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
streamQueryStatistics.IndexTimestamp = timeStamp;
}
}
public override async Task <ResponseDisposeHandling> ProcessResponse(JsonOperationContext context, HttpCache cache, HttpResponseMessage response, string url)
{
var state = new JsonParserState();
Stream stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false);
using (context.GetMemoryBuffer(out var buffer))
using (var parser = new UnmanagedJsonParser(context, state, "attachments/receive"))
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "attachments/list", parser, state))
using (var peepingTomStream = new PeepingTomStream(stream, context))
{
if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
{
throw new Exception("cannot parse stream");
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
throw new Exception($"Expected token {nameof(JsonParserToken.StartObject)}, but got {nameof(state.CurrentTokenType)}.");
}
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);
using (var data = builder.CreateReader())
{
if (data.TryGetMember(nameof(AttachmentsMetadata), out object obj) && obj is BlittableJsonReaderArray bjra)
{
foreach (BlittableJsonReaderObject e in bjra)
{
var cur = JsonDeserializationClient.AttachmentDetails(e);
AttachmentsMetadata.Add(cur);
}
}
}
var bufferSize = parser.BufferSize - parser.BufferOffset;
var copy = ArrayPool <byte> .Shared.Rent(bufferSize);
var copyMemory = new Memory <byte>(copy);
buffer.Memory.Memory.Slice(parser.BufferOffset, bufferSize).CopyTo(copyMemory);
Result = Iterate(stream, copy, bufferSize).GetEnumerator();
}
return(ResponseDisposeHandling.Manually);
}
private unsafe string ReadType()
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of object when reading type", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndObject)
{
return(null);
}
if (_state.CurrentTokenType != JsonParserToken.String)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected property type to be string, but was " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
return(_context.AllocateStringValue(null, _state.StringBuffer, _state.StringSize).ToString());
}
public async ValueTask<bool> MoveNextAsync()
{
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
_done = true;
_current = null;
return false;
}
using (var builder = new BlittableJsonDocumentBuilder(_context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", _parser, _state))
{
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false);
_current = builder.CreateReader();
return true;
}
}
public bool MoveNext()
{
AssertInitialized();
CheckIfContextOrCacheNeedToBeRenewed();
_timeSeriesIt?.Dispose();
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType != JsonParserToken.EndObject)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
return false;
}
_builder.Renew("readArray/singleResult", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
if (_isTimeSeriesStream)
UnmanagedJsonParserHelper.ReadProperty(_builder, _peepingTomStream, _parser, _buffer);
else
UnmanagedJsonParserHelper.ReadObject(_builder, _peepingTomStream, _parser, _buffer);
Current = _builder.CreateReader();
_builder.Reset();
if (_isTimeSeriesStream)
{
_timeSeriesIt = new TimeSeriesStreamEnumerator(_builderContext, _peepingTomStream, _parser, _state, _buffer);
_timeSeriesIt.Initialize();
}
return true;
}
public async ValueTask<bool> MoveNextAsync()
{
AssertInitialized();
CheckIfContextOrCacheNeedToBeRenewed();
if (_timeSeriesIt != null)
await _timeSeriesIt.DisposeAsync().ConfigureAwait(false);
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType != JsonParserToken.EndObject)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
return false;
}
_builder.Renew("readArray/singleResult", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
if (_isTimeSeriesStream)
await UnmanagedJsonParserHelper.ReadPropertyAsync(_builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false);
else
await UnmanagedJsonParserHelper.ReadObjectAsync(_builder, _peepingTomStream, _parser, _buffer, _token).ConfigureAwait(false);
Current = _builder.CreateReader();
_builder.Reset();
if (_isTimeSeriesStream)
{
_timeSeriesIt = new TimeSeriesStreamEnumerator(_builderContext, _peepingTomStream, _parser, _state, _buffer);
await _timeSeriesIt.InitializeAsync().ConfigureAwait(false);
}
return true;
}
public async Task InitializeAsync()
{
AssertNotSync();
try
{
_initialized = true;
_state = new JsonParserState();
_parser = new UnmanagedJsonParser(_session.Context, _state, "stream contents");
_builder = new BlittableJsonDocumentBuilder(_builderContext, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state);
_returnBuffer = _session.Context.GetMemoryBuffer(out _buffer);
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType != JsonParserToken.StartObject)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_isQueryStream)
HandleStreamQueryStats(_builderContext, _response, _parser, _state, _buffer, _streamQueryStatistics);
var property = UnmanagedJsonParserHelper.ReadString(_builderContext, _peepingTomStream, _parser, _state, _buffer);
if (string.Equals(property, "Results") == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (await UnmanagedJsonParserHelper.ReadAsync(_peepingTomStream, _parser, _state, _buffer, _token).ConfigureAwait(false) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
if (_state.CurrentTokenType != JsonParserToken.StartArray)
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
catch
{
await DisposeAsync().ConfigureAwait(false);
throw;
}
}
public bool MoveNext()
{
AssertInitialized();
CheckIfContextNeedsToBeRenewed();
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
if (_state.CurrentTokenType != JsonParserToken.EndObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(_peepingTomStream);
}
return(false);
}
using (var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "readArray/singleResult", _parser, _state))
{
if (_cachedItemsRenew == false)
{
_cachedItemsRenew = builder.NeedResetPropertiesCache();
}
UnmanagedJsonParserHelper.ReadObject(builder, _peepingTomStream, _parser, _buffer);
Current = builder.CreateReader();
return(true);
}
}
private void ReadObject(Action <BlittableJsonReaderObject> readAction)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
using (var builder = CreateBuilder(_context, null))
{
_context.CachedProperties.NewDocument();
ReadObject(builder);
using (var reader = builder.CreateReader())
{
readAction(reader);
}
}
}
private async Task ProcessChanges()
{
using (_requestExecutor.ContextPool.AllocateOperationContext(out var context))
{
while (_cts.IsCancellationRequested == false)
{
var state = new JsonParserState();
using (var stream = new WebSocketStream(_client, _cts.Token))
using (context.GetManagedBuffer(out var buffer))
using (var parser = new UnmanagedJsonParser(context, state, "changes/receive"))
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", parser, state))
using (var peepingTomStream = new PeepingTomStream(stream, context))
{
if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
{
continue;
}
if (state.CurrentTokenType != JsonParserToken.StartArray)
{
continue;
}
while (true)
{
if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
{
continue;
}
if (state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
builder.Renew("changes/receive", BlittableJsonDocumentBuilder.UsageMode.None);
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);
var json = builder.CreateReader();
try
{
if (json.TryGet("Type", out string type) == false)
{
continue;
}
switch (type)
{
case "Error":
json.TryGet("Exception", out string exceptionAsString);
NotifyAboutError(new Exception(exceptionAsString));
break;
case "Confirm":
if (json.TryGet("CommandId", out int commandId) &&
_confirmations.TryRemove(commandId, out var tcs))
{
tcs.TrySetResult(null);
}
break;
default:
json.TryGet("Value", out BlittableJsonReaderObject value);
NotifySubscribers(type, value, _counters.ValuesSnapshot);
break;
}
}
catch (Exception e)
{
NotifyAboutError(e);
throw new ChangeProcessingException(e);
}
}
}
}
}
}
public async Task ConnectInternal()
{
try
{
var urlBuilder = new StringBuilder(_url).Append("/admin/traffic-watch");
if (string.IsNullOrWhiteSpace(_database) == false)
{
urlBuilder.Append("?resourceName=").Append(_database);
}
var stringUrl = ToWebSocketPath(urlBuilder.ToString().ToLower());
var url = new Uri(stringUrl, UriKind.Absolute);
_client = new ClientWebSocket();
if (_cert != null)
{
_client.Options.ClientCertificates.Add(_cert);
}
await _client.ConnectAsync(url, _cancellationTokenSource.Token).ConfigureAwait(false);
_firstConnection = false;
Console.WriteLine($"Connected to RavenDB server. Collecting traffic watch entries to {_path}");
const int maxFileSize = 128 * 1024 * 1024;
while (_cancellationTokenSource.IsCancellationRequested == false)
{
string file = Path.Combine(_path, _logNameCreator.GetNewFileName());
var state = new JsonParserState();
using (var context = JsonOperationContext.ShortTermSingleUse())
// Read
await using (var stream = new WebSocketStream(_client, _cancellationTokenSource.Token))
using (context.GetMemoryBuffer(out var buffer))
using (var parser = new UnmanagedJsonParser(context, state, "trafficwatch/receive"))
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readObject/singleResult", parser, state))
// Write
await using (var fileStream = new FileStream(file, FileMode.Append, FileAccess.Write, FileShare.Read, 32 * 1024, false))
await using (var gZipStream = new GZipStream(fileStream, CompressionMode.Compress, false))
using (var peepingTomStream = new PeepingTomStream(stream, context))
await using (var writer = new AsyncBlittableJsonTextWriter(context, gZipStream))
{
writer.WriteStartArray();
var isFirst = true;
while (fileStream.Length < maxFileSize)
{
if (_cancellationTokenSource.IsCancellationRequested)
{
writer.WriteEndArray();
break;
}
try
{
var flushCount = 0;
while (fileStream.Length < maxFileSize && _cancellationTokenSource.IsCancellationRequested == false)
{
builder.Reset();
builder.Renew("trafficwatch/receive", BlittableJsonDocumentBuilder.UsageMode.None);
if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
{
continue;
}
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);
using (var json = builder.CreateReader())
{
if (_changeTypes != null)
{
if (json.TryGet("Type", out TrafficWatchChangeType type) == false)
{
continue;
}
if (_changeTypes.Contains(type) == false)
{
continue;
}
}
if (_database != null)
{
if (json.TryGet("DatabaseName", out LazyStringValue databaseName) == false ||
_database.Equals(databaseName, StringComparison.OrdinalIgnoreCase) == false)
{
continue;
}
}
if (isFirst == false)
{
writer.WriteComma();
}
isFirst = false;
if (_verbose)
{
Console.WriteLine(json);
}
writer.WriteObject(json);
_errorCount = 0;
if (flushCount++ % 128 == 0)
{
await writer.FlushAsync();
}
}
}
}
catch (Exception)
{
writer.WriteEndArray();
throw;
}
}
}
}
}
catch (ObjectDisposedException)
{
// closing
}
}
internal static IEnumerable <ReplayProgress> Replay(DocumentDatabase database, Stream replayStream)
{
DocumentsOperationContext txCtx = null;
IDisposable txDisposable = null;
DocumentsTransaction previousTx = null;
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.GetManagedBuffer(out var buffer))
using (var gZipStream = new GZipStream(replayStream, CompressionMode.Decompress, leaveOpen: true))
{
var peepingTomStream = new PeepingTomStream(gZipStream, context);
var state = new JsonParserState();
var parser = new UnmanagedJsonParser(context, state, "file");
var commandsProgress = 0;
var readers = UnmanagedJsonParserHelper.ReadArrayToMemory(context, peepingTomStream, parser, state, buffer);
using (var readersItr = readers.GetEnumerator())
{
ReadStartRecordingDetails(readersItr, context, peepingTomStream);
while (readersItr.MoveNext())
{
using (readersItr.Current)
{
if (readersItr.Current.TryGet(nameof(RecordingDetails.Type), out string strType) == false)
{
throw new ReplayTransactionsException($"Can't read {nameof(RecordingDetails.Type)} of replay detail", peepingTomStream);
}
if (Enum.TryParse <TxInstruction>(strType, true, out var type))
{
switch (type)
{
case TxInstruction.BeginTx:
txDisposable = database.DocumentsStorage.ContextPool.AllocateOperationContext(out txCtx);
txCtx.OpenWriteTransaction();
break;
case TxInstruction.Commit:
txCtx.Transaction.Commit();
break;
case TxInstruction.DisposeTx:
txDisposable.Dispose();
break;
case TxInstruction.BeginAsyncCommitAndStartNewTransaction:
previousTx = txCtx.Transaction;
txCtx.Transaction = txCtx.Transaction.BeginAsyncCommitAndStartNewTransaction(txCtx);
txDisposable = txCtx.Transaction;
break;
case TxInstruction.EndAsyncCommit:
previousTx.EndAsyncCommit();
break;
case TxInstruction.DisposePrevTx:
previousTx.Dispose();
break;
}
continue;
}
try
{
var cmd = DeserializeCommand(context, database, strType, readersItr.Current, peepingTomStream);
commandsProgress += cmd.ExecuteDirectly(txCtx);
TransactionOperationsMerger.UpdateGlobalReplicationInfoBeforeCommit(txCtx);
}
catch (Exception)
{
//TODO To accept exceptions that was thrown while recording
txDisposable.Dispose();
throw;
}
yield return(new ReplayProgress
{
CommandsProgress = commandsProgress
});
}
}
}
}
}
private static unsafe GetResponse ReadResponse(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer)
{
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
var getResponse = new GetResponse();
while (true)
{
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType == JsonParserToken.EndObject)
{
break;
}
if (state.CurrentTokenType != JsonParserToken.String)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
var property = context.AllocateStringValue(null, state.StringBuffer, state.StringSize).ToString();
switch (property)
{
case nameof(GetResponse.Result):
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType == JsonParserToken.Null)
{
continue;
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "multi_get/result", parser, state))
{
UnmanagedJsonParserHelper.ReadObject(builder, peepingTomStream, parser, buffer);
getResponse.Result = builder.CreateReader();
}
continue;
case nameof(GetResponse.Headers):
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType == JsonParserToken.Null)
{
continue;
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "multi_get/result", parser, state))
{
UnmanagedJsonParserHelper.ReadObject(builder, peepingTomStream, parser, buffer);
using (var headersJson = builder.CreateReader())
{
foreach (var propertyName in headersJson.GetPropertyNames())
{
getResponse.Headers[propertyName] = headersJson[propertyName].ToString();
}
}
}
continue;
case nameof(GetResponse.StatusCode):
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.Integer)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
getResponse.StatusCode = (HttpStatusCode)state.Long;
continue;
default:
ThrowInvalidJsonResponse(peepingTomStream);
break;
}
}
return(getResponse);
}
private IEnumerable <DocumentConflict> ReadConflicts(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var builder = CreateBuilder(context, null);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, null);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var conflict = new DocumentConflict();
if (data.TryGet(nameof(DocumentConflict.Id), out conflict.Id) &&
data.TryGet(nameof(DocumentConflict.Collection), out conflict.Collection) &&
data.TryGet(nameof(DocumentConflict.Flags), out string flags) &&
data.TryGet(nameof(DocumentConflict.ChangeVector), out conflict.ChangeVector) &&
data.TryGet(nameof(DocumentConflict.Etag), out conflict.Etag) &&
data.TryGet(nameof(DocumentConflict.LastModified), out conflict.LastModified) &&
data.TryGet(nameof(DocumentConflict.Doc), out conflict.Doc))
{
conflict.Flags = Enum.Parse <DocumentFlags>(flags);
if (conflict.Doc != null) // This is null for conflict that was generated from tombstone
{
conflict.Doc = context.ReadObject(conflict.Doc, conflict.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
yield return(conflict);
}
else
{
var msg = "Ignoring an invalid conflict which you try to import. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.Conflicts.ErroredCount++;
_result.AddWarning(msg);
}
}
}
finally
{
builder.Dispose();
}
}
private IEnumerable <Tombstone> ReadTombstones(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var builder = CreateBuilder(context, null);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, null);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var tombstone = new Tombstone();
if (data.TryGet("Key", out tombstone.LowerId) &&
data.TryGet(nameof(Tombstone.Type), out string type) &&
data.TryGet(nameof(Tombstone.Collection), out tombstone.Collection) &&
data.TryGet(nameof(Tombstone.LastModified), out tombstone.LastModified))
{
if (Enum.TryParse <Tombstone.TombstoneType>(type, out var tombstoneType) == false)
{
var msg = $"Ignoring a tombstone of type `{type}` which is not supported in 4.0. ";
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.Tombstones.ErroredCount++;
_result.AddWarning(msg);
continue;
}
tombstone.Type = tombstoneType;
yield return(tombstone);
}
else
{
var msg = "Ignoring an invalid tombstone which you try to import. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.Tombstones.ErroredCount++;
_result.AddWarning(msg);
}
}
}
finally
{
builder.Dispose();
}
}
public void JsonDeserialize_WhenHasBlittableObjectPropertyAndWriteAndReadFromStream_ShouldResultInCommandWithTheProperty()
{
using (Server.ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var jsonSerializer = new JsonSerializer
{
ContractResolver = new DefaultRavenContractResolver(DocumentConventions.Default.Serialization),
};
jsonSerializer.Converters.Add(BlittableJsonConverter.Instance);
var data = new { SomeProperty = "SomeValue" };
var expected = DocumentConventions.Default.Serialization.DefaultConverter.ToBlittable(data, context);
var command = new Command {
BlittableObject = expected
};
//Serialize
BlittableJsonReaderObject toStream;
using (var writer = new BlittableJsonWriter(context))
{
jsonSerializer.Serialize(writer, command);
writer.FinalizeDocument();
toStream = writer.CreateReader();
}
//Simulates copying to file and loading
BlittableJsonReaderObject fromStream;
using (Stream stream = new MemoryStream())
{
//Pass to stream
using (var textWriter = new BlittableJsonTextWriter(context, stream))
{
context.Write(textWriter, toStream);
}
//Get from stream
stream.Position = 0;
var state = new JsonParserState();
var parser = new UnmanagedJsonParser(context, state, "some tag");
var peepingTomStream = new PeepingTomStream(stream, context);
using (context.GetMemoryBuffer(out var buffer))
using (var builder =
new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "some tag", parser, state))
{
UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer);
UnmanagedJsonParserHelper.ReadObject(builder, peepingTomStream, parser, buffer);
fromStream = builder.CreateReader();
}
}
//Deserialize
BlittableJsonReaderObject actual;
using (var reader = new BlittableJsonReader(context))
{
reader.Initialize(fromStream);
var deserialized = jsonSerializer.Deserialize <Command>(reader);
actual = deserialized.BlittableObject;
}
Assert.Equal(expected, actual);
}
}
private async Task ProcessChanges()
{
using (_requestExecutor.ContextPool.AllocateOperationContext(out var context))
{
while (_cts.IsCancellationRequested == false)
{
context.Reset();
context.Renew();
var state = new JsonParserState();
using (var stream = new WebSocketStream(_client, _cts.Token))
using (context.GetMemoryBuffer(out JsonOperationContext.MemoryBuffer buffer))
using (var parser = new UnmanagedJsonParser(context, state, "changes/receive"))
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", parser, state))
using (var peepingTomStream = new PeepingTomStream(stream, context))
{
if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
{
continue;
}
if (state.CurrentTokenType != JsonParserToken.StartArray)
{
continue;
}
while (true)
{
builder.Reset();
builder.Renew("changes/receive", BlittableJsonDocumentBuilder.UsageMode.None);
if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
{
continue;
}
if (state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);
using (var json = builder.CreateReader())
{
try
{
if (json.TryGet(nameof(TopologyChange), out bool supports) && supports)
{
GetOrAddConnectionState("Topology", "watch-topology-change", "", "");
await _requestExecutor.UpdateTopologyAsync(new RequestExecutor.UpdateTopologyParameters(_serverNode)
{
TimeoutInMs = 0, ForceUpdate = true, DebugTag = "watch-topology-change"
}).ConfigureAwait(false);
continue;
}
if (json.TryGet("Type", out string type) == false)
{
continue;
}
switch (type)
{
case "Error":
json.TryGet("Exception", out string exceptionAsString);
NotifyAboutError(new Exception(exceptionAsString));
break;
case "Confirm":
if (json.TryGet("CommandId", out int commandId) &&
_confirmations.TryRemove(commandId, out var tcs))
{
tcs.TrySetResult(null);
}
break;
default:
json.TryGet("Value", out BlittableJsonReaderObject value);
NotifySubscribers(type, value, _counters.ForceEnumerateInThreadSafeManner().Select(x => x.Value).ToList());
break;
}
}
catch (Exception e)
{
NotifyAboutError(e);
throw new ChangeProcessingException(e);
}
}
}
}
}
}
}
private IEnumerable <DocumentItem> ReadDocuments(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var legacyImport = _buildVersionType == BuildVersionType.V3;
var modifier = new BlittableMetadataModifier(context)
{
ReadFirstEtagOfLegacyRevision = legacyImport,
ReadLegacyEtag = _readLegacyEtag,
OperateOnTypes = _operateOnTypes
};
var builder = CreateBuilder(context, modifier);
try
{
List <DocumentItem.AttachmentStream> attachments = null;
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, modifier);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type))
{
if (type != DocumentItem.ExportDocumentType.Attachment)
{
var msg = $"Ignoring an item of type `{type}`. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.AddWarning(msg);
continue;
}
if (attachments == null)
{
attachments = new List <DocumentItem.AttachmentStream>();
}
var attachment = new DocumentItem.AttachmentStream
{
Stream = actions.GetTempStream()
};
ProcessAttachmentStream(context, data, ref attachment);
attachments.Add(attachment);
continue;
}
if (legacyImport)
{
if (modifier.Id.Contains(HiLoHandler.RavenHiloIdPrefix))
{
data.Modifications = new DynamicJsonValue
{
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection
}
};
}
}
if (data.Modifications != null)
{
data = context.ReadObject(data, modifier.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
_result.LegacyLastDocumentEtag = modifier.LegacyEtag;
yield return(new DocumentItem
{
Document = new Document
{
Data = data,
Id = modifier.Id,
ChangeVector = modifier.ChangeVector,
Flags = modifier.Flags,
NonPersistentFlags = modifier.NonPersistentFlags,
LastModified = modifier.LastModified ?? _database.Time.GetUtcNow(),
},
Attachments = attachments
});
attachments = null;
}
}
finally
{
builder.Dispose();
}
}
