public StreamSource(Stream stream, DocumentsOperationContext context, DocumentDatabase database)
{
_peepingTomStream = new PeepingTomStream(stream, context);
_context = context;
_database = database;
_log = LoggingSource.Instance.GetLogger <StreamSource>(database.Name);
}
public YieldStreamResults(InMemoryDocumentSessionOperations session, StreamResult stream, bool isQueryStream)
{
_response = stream;
_peepingTomStream = new PeepingTomStream(_response.Stream, session.Context);
_session = session;
_isQueryStream = isQueryStream;
}
public static async Task <BlittableJsonReaderObject> SimulateSavingToFileAndLoadingAsync(JsonOperationContext context, BlittableJsonReaderObject toStream)
{
//Simulates saving to file and loading
BlittableJsonReaderObject fromStream;
await using (Stream stream = new MemoryStream())
{
//Pass to stream
await using (var textWriter = new AsyncBlittableJsonTextWriter(context, stream))
{
context.Write(textWriter, toStream);
}
//Get from stream
stream.Position = 0;
var state = new JsonParserState();
var parser = new UnmanagedJsonParser(context, state, "some tag");
var peepingTomStream = new PeepingTomStream(stream, context);
using (context.GetMemoryBuffer(out var buffer))
using (var builder =
new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "some tag", parser, state))
{
await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer);
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer);
fromStream = builder.CreateReader();
}
}
return(fromStream);
}
public TimeSeriesStreamEnumerator(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, CancellationToken token = default)
{
_context = context;
_peepingTomStream = peepingTomStream;
_parser = parser;
_state = state;
_buffer = buffer;
_token = token;
}
public TimeSeriesStreamEnumerator(InMemoryDocumentSessionOperations session, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, CancellationToken token = default)
{
_session = session;
_peepingTomStream = peepingTomStream;
_parser = parser;
_state = state;
_buffer = buffer;
_token = token;
}
public void PeepingTomStreamTest(int originalSize, int chunkSizeToRead, int offset, JsonOperationContext context)
{
var bytes = new byte[originalSize];
for (var i = 0; i < bytes.Length; i++)
{
bytes[i] = (byte)((i % 26) + 'a');
}
using (var stream = new MemoryStream())
using (var peeping = new PeepingTomStream(stream, context))
{
stream.Write(bytes, 0, originalSize);
stream.Flush();
stream.Position = 0;
var totalRead = 0;
do
{
int read;
do
{
var buffer = new byte[originalSize + offset];
read = peeping.Read(buffer, offset, chunkSizeToRead);
totalRead += read;
Assert.True(read <= chunkSizeToRead);
} while (read != 0);
} while (totalRead < originalSize);
Assert.Equal(originalSize, totalRead);
var peepWindow = peeping.PeepInReadStream();
var length = peepWindow.Length;
Assert.True(length <= PeepingTomStream.BufferWindowSize);
Assert.True(length >= 0);
var expectedLength = originalSize < PeepingTomStream.BufferWindowSize ? originalSize : PeepingTomStream.BufferWindowSize;
if (expectedLength != length)
{
var expected = System.Text.Encoding.UTF8.GetString(bytes, bytes.Length - expectedLength, expectedLength);
var actual = System.Text.Encoding.UTF8.GetString(peepWindow);
Assert.Equal(expected, actual);
}
Assert.Equal(expectedLength, length);
for (var i = 0; i < peepWindow.Length; i++)
{
var expectedByte = (byte)(((originalSize - peepWindow.Length + i) % 26) + 'a');
if (expectedByte != peepWindow[i])
{
Assert.Equal(expectedByte, peepWindow[i]);
}
}
}
}
public IEnumerator <BlittableJsonReaderObject> SetResult(StreamResult response)
{
if (response == null)
{
throw new InvalidOperationException("The index does not exists, failed to stream results");
}
var state = new JsonParserState();
JsonOperationContext.ManagedPinnedBuffer buffer;
using (response.Response)
using (response.Stream)
using (var parser = new UnmanagedJsonParser(_session.Context, state, "stream contents"))
using (_session.Context.GetManagedBuffer(out buffer))
using (var peepingTomStream = new PeepingTomStream(response.Stream, _session.Context))
{
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
if (_isQueryStream)
{
HandleStreamQueryStats(_session.Context, response, parser, state, buffer, _statistics);
}
var property = UnmanagedJsonParserHelper.ReadString(_session.Context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, "Results") == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
foreach (var result in UnmanagedJsonParserHelper.ReadArrayToMemory(_session.Context, peepingTomStream, parser, state, buffer))
{
yield return(result);
}
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
}
}
}
public YieldStreamResults(InMemoryDocumentSessionOperations session, StreamResult response, bool isQueryStream, bool isTimeSeriesStream, bool isAsync, StreamQueryStatistics streamQueryStatistics, CancellationToken token = default)
{
_response = response ?? throw new InvalidOperationException("The index does not exists, failed to stream results");
_peepingTomStream = new PeepingTomStream(_response.Stream, session.Context);
_session = session;
_isQueryStream = isQueryStream;
_isAsync = isAsync;
_streamQueryStatistics = streamQueryStatistics;
_maxDocsCountOnCachedRenewSession = session._maxDocsCountOnCachedRenewSession;
_token = token;
_isTimeSeriesStream = isTimeSeriesStream;
}
public override void SetResponseRaw(HttpResponseMessage response, Stream stream, JsonOperationContext context)
{
var state = new JsonParserState();
using (var parser = new UnmanagedJsonParser(context, state, "multi_get/response"))
using (context.GetMemoryBuffer(out var buffer))
using (var peepingTomStream = new PeepingTomStream(stream, context))
using (_cached)
{
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (property != nameof(BlittableArrayResult.Results))
{
ThrowInvalidJsonResponse(peepingTomStream);
}
var i = 0;
Result = new List <GetResponse>(_commands.Count);
foreach (var getResponse in ReadResponses(context, peepingTomStream, parser, state, buffer))
{
var command = _commands[i];
MaybeSetCache(getResponse, command);
Result.Add(_cached != null && getResponse.StatusCode == HttpStatusCode.NotModified
? new GetResponse {
Result = _cached.Values[i].Cached.Clone(context), StatusCode = HttpStatusCode.NotModified
}
: getResponse);
i++;
}
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
}
}
public ReplayTransactionsException(string message, PeepingTomStream peepingTomStream)
: base(message)
{
try
{
Context = Encodings.Utf8.GetString(peepingTomStream.PeepInReadStream());
}
catch (Exception e)
{
Context = "Failed to generated peepedWindow: " + e;
}
}
public static unsafe string ReadString(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer)
{
if (Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.String)
{
ThrowInvalidJson(peepingTomStream);
}
return(context.AllocateStringValue(null, state.StringBuffer, state.StringSize).ToString());
}
private static string GetPeepingTomBufferAsString(PeepingTomStream peepingTomStream)
{
string peepedWindow;
try
{
peepedWindow = Encodings.Utf8.GetString(peepingTomStream.PeepInReadStream());
}
catch (Exception e)
{
peepedWindow = "Failed to generated peepedWindow: " + e;
}
return(peepedWindow);
}
public static long ReadLong(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer)
{
if (Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.Integer)
{
ThrowInvalidJson(peepingTomStream);
}
return(state.Long);
}
public YieldStreamResults(InMemoryDocumentSessionOperations session, StreamResult response, bool isQueryStream, bool isAsync, StreamQueryStatistics streamQueryStatistics)
{
if (response == null)
{
throw new InvalidOperationException("The index does not exists, failed to stream results");
}
_response = response;
_peepingTomStream = new PeepingTomStream(_response.Stream, session.Context);
_session = session;
_isQueryStream = isQueryStream;
_isAsync = isAsync;
_streamQueryStatistics = streamQueryStatistics;
}
public override void SetResponseRaw(HttpResponseMessage response, Stream stream, JsonOperationContext context)
{
var state = new JsonParserState();
using (var parser = new UnmanagedJsonParser(context, state, "multi_get/response"))
using (context.GetManagedBuffer(out JsonOperationContext.ManagedPinnedBuffer buffer))
using (var peepingTomStream = new PeepingTomStream(stream, context))
{
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (property != nameof(BlittableArrayResult.Results))
{
ThrowInvalidJsonResponse(peepingTomStream);
}
var i = 0;
Result = new List <GetResponse>();
foreach (var getResponse in ReadResponses(context, peepingTomStream, parser, state, buffer))
{
var command = _commands[i];
MaybeSetCache(getResponse, command);
MaybeReadFromCache(getResponse, command, context);
Result.Add(getResponse);
i++;
}
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
}
}
private static void HandleStreamQueryStats(JsonOperationContext context, StreamResult response, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, StreamQueryStatistics streamQueryStatistics = null)
{
using (var peepingTomStream = new PeepingTomStream(response.Stream, context))
{
var property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.ResultEtag)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var resultEtag = UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer);
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.IsStale)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
if (state.CurrentTokenType != JsonParserToken.False && state.CurrentTokenType != JsonParserToken.True)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var isStale = state.CurrentTokenType != JsonParserToken.False;
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.IndexName)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var indexName = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.TotalResults)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var totalResults = (int)UnmanagedJsonParserHelper.ReadLong(context, peepingTomStream, parser, state, buffer);
property = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (string.Equals(property, nameof(StreamQueryStatistics.IndexTimestamp)) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
var indexTimestamp = UnmanagedJsonParserHelper.ReadString(context, peepingTomStream, parser, state, buffer);
if (streamQueryStatistics == null)
return;
streamQueryStatistics.IndexName = indexName;
streamQueryStatistics.IsStale = isStale;
streamQueryStatistics.TotalResults = totalResults;
streamQueryStatistics.ResultEtag = resultEtag;
if (DateTime.TryParseExact(indexTimestamp, "o", CultureInfo.InvariantCulture,
DateTimeStyles.RoundtripKind, out DateTime timeStamp) == false)
UnmanagedJsonParserHelper.ThrowInvalidJson(peepingTomStream);
streamQueryStatistics.IndexTimestamp = timeStamp;
}
}
public static void ReadObject(BlittableJsonDocumentBuilder builder, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonOperationContext.ManagedPinnedBuffer buffer)
{
builder.ReadNestedObject();
while (builder.Read() == false)
{
var read = peepingTomStream.Read(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content" + GetPeepingTomBufferAsString(peepingTomStream));
}
parser.SetBuffer(buffer, 0, read);
}
builder.FinalizeDocument();
}
public static void ReadProperty(BlittableJsonDocumentBuilder builder, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonOperationContext.MemoryBuffer buffer)
{
builder.ReadProperty();
while (builder.Read() == false)
{
var read = peepingTomStream.Read(buffer.Memory.Memory.Span);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content" + GetPeepingTomBufferAsString(peepingTomStream));
}
parser.SetBuffer(buffer, 0, read);
}
builder.FinalizeDocument();
}
public override async Task <ResponseDisposeHandling> ProcessResponse(JsonOperationContext context, HttpCache cache, HttpResponseMessage response, string url)
{
var state = new JsonParserState();
Stream stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false);
using (context.GetMemoryBuffer(out var buffer))
using (var parser = new UnmanagedJsonParser(context, state, "attachments/receive"))
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "attachments/list", parser, state))
using (var peepingTomStream = new PeepingTomStream(stream, context))
{
if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
{
throw new Exception("cannot parse stream");
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
throw new Exception($"Expected token {nameof(JsonParserToken.StartObject)}, but got {nameof(state.CurrentTokenType)}.");
}
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);
using (var data = builder.CreateReader())
{
if (data.TryGetMember(nameof(AttachmentsMetadata), out object obj) && obj is BlittableJsonReaderArray bjra)
{
foreach (BlittableJsonReaderObject e in bjra)
{
var cur = JsonDeserializationClient.AttachmentDetails(e);
AttachmentsMetadata.Add(cur);
}
}
}
var bufferSize = parser.BufferSize - parser.BufferOffset;
var copy = ArrayPool <byte> .Shared.Rent(bufferSize);
var copyMemory = new Memory <byte>(copy);
buffer.Memory.Memory.Slice(parser.BufferOffset, bufferSize).CopyTo(copyMemory);
Result = Iterate(stream, copy, bufferSize).GetEnumerator();
}
return(ResponseDisposeHandling.Manually);
}
public static bool Read(PeepingTomStream stream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer)
{
while (parser.Read() == false)
{
var read = stream.Read(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length);
if (read == 0)
{
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
return(false);
}
parser.SetBuffer(buffer, 0, read);
}
return(true);
}
public static async Task <bool> ReadAsync(PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer, CancellationToken token = default)
{
while (parser.Read() == false)
{
var read = await peepingTomStream.ReadAsync(buffer.Memory.Memory, token).ConfigureAwait(false);
if (read == 0)
{
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
return(false);
}
parser.SetBuffer(buffer, 0, read);
}
return(true);
}
private static TransactionOperationsMerger.MergedTransactionCommand DeserializeCommand(
DocumentsOperationContext context,
DocumentDatabase database,
string type,
BlittableJsonReaderObject wrapCmdReader,
PeepingTomStream peepingTomStream)
{
if (!wrapCmdReader.TryGet(nameof(RecordingCommandDetails.Command), out BlittableJsonReaderObject commandReader))
{
throw new ReplayTransactionsException($"Can't read {type} for replay", peepingTomStream);
}
var jsonSerializer = GetJsonSerializer();
using (var reader = new BlittableJsonReader(context))
{
reader.Init(commandReader);
var dto = DeserializeCommandDto(type, jsonSerializer, reader, peepingTomStream);
return(dto.ToCommand(context, database));
}
}
public static async Task <bool> ReadAsync(PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer)
{
if (parser.Read())
{
return(true);
}
var read = await peepingTomStream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length).ConfigureAwait(false);
if (read == 0)
{
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
return(false);
}
parser.SetBuffer(buffer, 0, read);
return(parser.Read());
}
private async Task ProcessChanges()
{
using (_requestExecutor.ContextPool.AllocateOperationContext(out var context))
{
while (_cts.IsCancellationRequested == false)
{
context.Reset();
context.Renew();
var state = new JsonParserState();
using (var stream = new WebSocketStream(_client, _cts.Token))
using (context.GetMemoryBuffer(out JsonOperationContext.MemoryBuffer buffer))
using (var parser = new UnmanagedJsonParser(context, state, "changes/receive"))
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", parser, state))
using (var peepingTomStream = new PeepingTomStream(stream, context))
{
if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
{
continue;
}
if (state.CurrentTokenType != JsonParserToken.StartArray)
{
continue;
}
while (true)
{
builder.Reset();
builder.Renew("changes/receive", BlittableJsonDocumentBuilder.UsageMode.None);
if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
{
continue;
}
if (state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);
using (var json = builder.CreateReader())
{
try
{
if (json.TryGet(nameof(TopologyChange), out bool supports) && supports)
{
GetOrAddConnectionState("Topology", "watch-topology-change", "", "");
await _requestExecutor.UpdateTopologyAsync(new RequestExecutor.UpdateTopologyParameters(_serverNode)
{
TimeoutInMs = 0, ForceUpdate = true, DebugTag = "watch-topology-change"
}).ConfigureAwait(false);
continue;
}
if (json.TryGet("Type", out string type) == false)
{
continue;
}
switch (type)
{
case "Error":
json.TryGet("Exception", out string exceptionAsString);
NotifyAboutError(new Exception(exceptionAsString));
break;
case "Confirm":
if (json.TryGet("CommandId", out int commandId) &&
_confirmations.TryRemove(commandId, out var tcs))
{
tcs.TrySetResult(null);
}
break;
default:
json.TryGet("Value", out BlittableJsonReaderObject value);
NotifySubscribers(type, value, _counters.ForceEnumerateInThreadSafeManner().Select(x => x.Value).ToList());
break;
}
}
catch (Exception e)
{
NotifyAboutError(e);
throw new ChangeProcessingException(e);
}
}
}
}
}
}
}
public void JsonDeserialize_WhenHasBlittableObjectPropertyAndWriteAndReadFromStream_ShouldResultInCommandWithTheProperty()
{
using (Server.ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var jsonSerializer = new JsonSerializer
{
ContractResolver = new DefaultRavenContractResolver(DocumentConventions.Default.Serialization),
};
jsonSerializer.Converters.Add(BlittableJsonConverter.Instance);
var data = new { SomeProperty = "SomeValue" };
var expected = DocumentConventions.Default.Serialization.DefaultConverter.ToBlittable(data, context);
var command = new Command {
BlittableObject = expected
};
//Serialize
BlittableJsonReaderObject toStream;
using (var writer = new BlittableJsonWriter(context))
{
jsonSerializer.Serialize(writer, command);
writer.FinalizeDocument();
toStream = writer.CreateReader();
}
//Simulates copying to file and loading
BlittableJsonReaderObject fromStream;
using (Stream stream = new MemoryStream())
{
//Pass to stream
using (var textWriter = new BlittableJsonTextWriter(context, stream))
{
context.Write(textWriter, toStream);
}
//Get from stream
stream.Position = 0;
var state = new JsonParserState();
var parser = new UnmanagedJsonParser(context, state, "some tag");
var peepingTomStream = new PeepingTomStream(stream, context);
using (context.GetMemoryBuffer(out var buffer))
using (var builder =
new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "some tag", parser, state))
{
UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer);
UnmanagedJsonParserHelper.ReadObject(builder, peepingTomStream, parser, buffer);
fromStream = builder.CreateReader();
}
}
//Deserialize
BlittableJsonReaderObject actual;
using (var reader = new BlittableJsonReader(context))
{
reader.Initialize(fromStream);
var deserialized = jsonSerializer.Deserialize <Command>(reader);
actual = deserialized.BlittableObject;
}
Assert.Equal(expected, actual);
}
}
private async Task ProcessChanges()
{
using (_requestExecutor.ContextPool.AllocateOperationContext(out var context))
{
while (_cts.IsCancellationRequested == false)
{
var state = new JsonParserState();
using (var stream = new WebSocketStream(_client, _cts.Token))
using (context.GetManagedBuffer(out var buffer))
using (var parser = new UnmanagedJsonParser(context, state, "changes/receive"))
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", parser, state))
using (var peepingTomStream = new PeepingTomStream(stream, context))
{
if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
{
continue;
}
if (state.CurrentTokenType != JsonParserToken.StartArray)
{
continue;
}
while (true)
{
if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
{
continue;
}
if (state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
builder.Renew("changes/receive", BlittableJsonDocumentBuilder.UsageMode.None);
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);
var json = builder.CreateReader();
try
{
if (json.TryGet("Type", out string type) == false)
{
continue;
}
switch (type)
{
case "Error":
json.TryGet("Exception", out string exceptionAsString);
NotifyAboutError(new Exception(exceptionAsString));
break;
case "Confirm":
if (json.TryGet("CommandId", out int commandId) &&
_confirmations.TryRemove(commandId, out var tcs))
{
tcs.TrySetResult(null);
}
break;
default:
json.TryGet("Value", out BlittableJsonReaderObject value);
NotifySubscribers(type, value, _counters.ValuesSnapshot);
break;
}
}
catch (Exception e)
{
NotifyAboutError(e);
throw new ChangeProcessingException(e);
}
}
}
}
}
}
public async Task ConnectInternal()
{
try
{
var urlBuilder = new StringBuilder(_url).Append("/admin/traffic-watch");
if (string.IsNullOrWhiteSpace(_database) == false)
{
urlBuilder.Append("?resourceName=").Append(_database);
}
var stringUrl = ToWebSocketPath(urlBuilder.ToString().ToLower());
var url = new Uri(stringUrl, UriKind.Absolute);
_client = new ClientWebSocket();
if (_cert != null)
{
_client.Options.ClientCertificates.Add(_cert);
}
await _client.ConnectAsync(url, _cancellationTokenSource.Token).ConfigureAwait(false);
_firstConnection = false;
Console.WriteLine($"Connected to RavenDB server. Collecting traffic watch entries to {_path}");
const int maxFileSize = 128 * 1024 * 1024;
while (_cancellationTokenSource.IsCancellationRequested == false)
{
string file = Path.Combine(_path, _logNameCreator.GetNewFileName());
var state = new JsonParserState();
using (var context = JsonOperationContext.ShortTermSingleUse())
// Read
await using (var stream = new WebSocketStream(_client, _cancellationTokenSource.Token))
using (context.GetMemoryBuffer(out var buffer))
using (var parser = new UnmanagedJsonParser(context, state, "trafficwatch/receive"))
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readObject/singleResult", parser, state))
// Write
await using (var fileStream = new FileStream(file, FileMode.Append, FileAccess.Write, FileShare.Read, 32 * 1024, false))
await using (var gZipStream = new GZipStream(fileStream, CompressionMode.Compress, false))
using (var peepingTomStream = new PeepingTomStream(stream, context))
await using (var writer = new AsyncBlittableJsonTextWriter(context, gZipStream))
{
writer.WriteStartArray();
var isFirst = true;
while (fileStream.Length < maxFileSize)
{
if (_cancellationTokenSource.IsCancellationRequested)
{
writer.WriteEndArray();
break;
}
try
{
var flushCount = 0;
while (fileStream.Length < maxFileSize && _cancellationTokenSource.IsCancellationRequested == false)
{
builder.Reset();
builder.Renew("trafficwatch/receive", BlittableJsonDocumentBuilder.UsageMode.None);
if (await UnmanagedJsonParserHelper.ReadAsync(peepingTomStream, parser, state, buffer).ConfigureAwait(false) == false)
{
continue;
}
await UnmanagedJsonParserHelper.ReadObjectAsync(builder, peepingTomStream, parser, buffer).ConfigureAwait(false);
using (var json = builder.CreateReader())
{
if (_changeTypes != null)
{
if (json.TryGet("Type", out TrafficWatchChangeType type) == false)
{
continue;
}
if (_changeTypes.Contains(type) == false)
{
continue;
}
}
if (_database != null)
{
if (json.TryGet("DatabaseName", out LazyStringValue databaseName) == false ||
_database.Equals(databaseName, StringComparison.OrdinalIgnoreCase) == false)
{
continue;
}
}
if (isFirst == false)
{
writer.WriteComma();
}
isFirst = false;
if (_verbose)
{
Console.WriteLine(json);
}
writer.WriteObject(json);
_errorCount = 0;
if (flushCount++ % 128 == 0)
{
await writer.FlushAsync();
}
}
}
}
catch (Exception)
{
writer.WriteEndArray();
throw;
}
}
}
}
}
catch (ObjectDisposedException)
{
// closing
}
}
private static void ReadStartRecordingDetails(IEnumerator <BlittableJsonReaderObject> iterator, DocumentsOperationContext context, PeepingTomStream peepingTomStream)
{
if (false == iterator.MoveNext())
{
throw new ReplayTransactionsException("Replay stream is empty", peepingTomStream);
}
using (iterator.Current)
{
var jsonSerializer = GetJsonSerializer();
StartRecordingDetails startDetail;
using (var reader = new BlittableJsonReader(context))
{
reader.Init(iterator.Current);
startDetail = jsonSerializer.Deserialize <StartRecordingDetails>(reader);
}
if (string.IsNullOrEmpty(startDetail.Type))
{
throw new ReplayTransactionsException($"Can't read {nameof(RecordingDetails.Type)} of replay detail", peepingTomStream);
}
if (string.IsNullOrEmpty(startDetail.Type))
{
throw new ReplayTransactionsException($"Can't read {nameof(StartRecordingDetails.Version)} of replay instructions", peepingTomStream);
}
if (startDetail.Version != RavenVersionAttribute.Instance.Build)
{
throw new ReplayTransactionsException($"Can't replay transaction instructions of different server version - Current version({ServerVersion.FullVersion}), Record version({startDetail.Version})", peepingTomStream);
}
}
}
internal static IEnumerable <ReplayProgress> Replay(DocumentDatabase database, Stream replayStream)
{
DocumentsOperationContext txCtx = null;
IDisposable txDisposable = null;
DocumentsTransaction previousTx = null;
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.GetManagedBuffer(out var buffer))
using (var gZipStream = new GZipStream(replayStream, CompressionMode.Decompress, leaveOpen: true))
{
var peepingTomStream = new PeepingTomStream(gZipStream, context);
var state = new JsonParserState();
var parser = new UnmanagedJsonParser(context, state, "file");
var commandsProgress = 0;
var readers = UnmanagedJsonParserHelper.ReadArrayToMemory(context, peepingTomStream, parser, state, buffer);
using (var readersItr = readers.GetEnumerator())
{
ReadStartRecordingDetails(readersItr, context, peepingTomStream);
while (readersItr.MoveNext())
{
using (readersItr.Current)
{
if (readersItr.Current.TryGet(nameof(RecordingDetails.Type), out string strType) == false)
{
throw new ReplayTransactionsException($"Can't read {nameof(RecordingDetails.Type)} of replay detail", peepingTomStream);
}
if (Enum.TryParse <TxInstruction>(strType, true, out var type))
{
switch (type)
{
case TxInstruction.BeginTx:
txDisposable = database.DocumentsStorage.ContextPool.AllocateOperationContext(out txCtx);
txCtx.OpenWriteTransaction();
break;
case TxInstruction.Commit:
txCtx.Transaction.Commit();
break;
case TxInstruction.DisposeTx:
txDisposable.Dispose();
break;
case TxInstruction.BeginAsyncCommitAndStartNewTransaction:
previousTx = txCtx.Transaction;
txCtx.Transaction = txCtx.Transaction.BeginAsyncCommitAndStartNewTransaction(txCtx);
txDisposable = txCtx.Transaction;
break;
case TxInstruction.EndAsyncCommit:
previousTx.EndAsyncCommit();
break;
case TxInstruction.DisposePrevTx:
previousTx.Dispose();
break;
}
continue;
}
try
{
var cmd = DeserializeCommand(context, database, strType, readersItr.Current, peepingTomStream);
commandsProgress += cmd.ExecuteDirectly(txCtx);
TransactionOperationsMerger.UpdateGlobalReplicationInfoBeforeCommit(txCtx);
}
catch (Exception)
{
//TODO To accept exceptions that was thrown while recording
txDisposable.Dispose();
throw;
}
yield return(new ReplayProgress
{
CommandsProgress = commandsProgress
});
}
}
}
}
}
private static TransactionOperationsMerger.IReplayableCommandDto <TransactionOperationsMerger.MergedTransactionCommand> DeserializeCommandDto(
string type,
JsonSerializer jsonSerializer,
BlittableJsonReader reader,
PeepingTomStream peepingTomStream)
{
switch (type)
{
case nameof(BatchHandler.MergedBatchCommand):
return(jsonSerializer.Deserialize <MergedBatchCommandDto>(reader));
case nameof(DeleteDocumentCommand):
return(jsonSerializer.Deserialize <DeleteDocumentCommandDto>(reader));
case nameof(PatchDocumentCommand):
return(jsonSerializer.Deserialize <PatchDocumentCommandDto>(reader));
case nameof(DatabaseDestination.MergedBatchPutCommand):
return(jsonSerializer.Deserialize <DatabaseDestination.MergedBatchPutCommandDto>(reader));
case nameof(MergedPutCommand):
return(jsonSerializer.Deserialize <MergedPutCommand.MergedPutCommandDto>(reader));
case nameof(BulkInsertHandler.MergedInsertBulkCommand):
return(jsonSerializer.Deserialize <MergedInsertBulkCommandDto>(reader));
case nameof(AttachmentHandler.MergedPutAttachmentCommand):
return(jsonSerializer.Deserialize <MergedPutAttachmentCommandDto>(reader));
case nameof(AttachmentHandler.MergedDeleteAttachmentCommand):
return(jsonSerializer.Deserialize <MergedDeleteAttachmentCommandDto>(reader));
case nameof(ResolveConflictOnReplicationConfigurationChange.PutResolvedConflictsCommand):
return(jsonSerializer.Deserialize <PutResolvedConflictsCommandDto>(reader));
case nameof(HiLoHandler.MergedNextHiLoCommand):
return(jsonSerializer.Deserialize <MergedNextHiLoCommandDto>(reader));
case nameof(HiLoHandler.MergedHiLoReturnCommand):
return(jsonSerializer.Deserialize <MergedHiLoReturnCommandDto>(reader));
case nameof(IncomingReplicationHandler.MergedDocumentReplicationCommand):
return(jsonSerializer.Deserialize <MergedDocumentReplicationCommandDto>(reader));
case nameof(ExpiredDocumentsCleaner.DeleteExpiredDocumentsCommand):
return(jsonSerializer.Deserialize <DeleteExpiredDocumentsCommandDto>(reader));
case nameof(OutgoingReplicationHandler.UpdateSiblingCurrentEtag):
return(jsonSerializer.Deserialize <UpdateSiblingCurrentEtagDto>(reader));
case nameof(IncomingReplicationHandler.MergedUpdateDatabaseChangeVectorCommand):
return(jsonSerializer.Deserialize <MergedUpdateDatabaseChangeVectorCommandDto>(reader));
case nameof(AdminRevisionsHandler.DeleteRevisionsCommand):
return(jsonSerializer.Deserialize <DeleteRevisionsCommandDto>(reader));
case nameof(RevisionsOperations.DeleteRevisionsBeforeCommand):
throw new ReplayTransactionsException(
"Because this command is deleting according to revisions' date & the revisions that created by replaying have different date an in place decision needed to be made",
peepingTomStream);
case nameof(TombstoneCleaner.DeleteTombstonesCommand):
return(jsonSerializer.Deserialize <DeleteTombstonesCommandDto>(reader));
case nameof(OutputReduceIndexWriteOperation.OutputReduceToCollectionCommand):
return(jsonSerializer.Deserialize <OutputReduceToCollectionCommandDto>(reader));
case nameof(BatchHandler.ClusterTransactionMergedCommand):
return(jsonSerializer.Deserialize <ClusterTransactionMergedCommandDto>(reader));
case nameof(CountersHandler.ExecuteCounterBatchCommand):
return(jsonSerializer.Deserialize <ExecuteCounterBatchCommandDto>(reader));
default:
throw new ReplayTransactionsException($"Can't read {type} for replay", peepingTomStream);
}
}
