public async Task <BlittableJsonReaderObject> TryReadFromWebSocket(
JsonOperationContext context,
RavenClientWebSocket webSocket,
string debugTag,
CancellationToken cancellationToken)
{
var jsonParserState = new JsonParserState();
JsonOperationContext.ManagedPinnedBuffer bytes;
using (context.GetManagedBuffer(out bytes))
using (var parser = new UnmanagedJsonParser(context, jsonParserState, debugTag))
{
var writer = new BlittableJsonDocumentBuilder(context,
BlittableJsonDocumentBuilder.UsageMode.None, debugTag, parser, jsonParserState);
writer.ReadObjectDocument();
var result = await webSocket.ReceiveAsync(bytes.Buffer, cancellationToken).ConfigureAwait(false);
parser.SetBuffer(bytes, result.Count);
while (writer.Read() == false)
{
// we got incomplete json response.
// This might happen if we close the connection but still server sends something
if (result.CloseStatus != null)
{
return(null);
}
result = await webSocket.ReceiveAsync(bytes.Buffer, cancellationToken).ConfigureAwait(false);
parser.SetBuffer(bytes, result.Count);
}
writer.FinalizeDocument();
return(writer.CreateReader());
}
}
public unsafe void Valid_String()
{
using (var ctx = JsonOperationContext.ShortTermSingleUse())
{
var state = new JsonParserState();
using (var parser = new UnmanagedJsonParser(ctx, state, "test"))
{
var temp = new Str
{
str = "\nabcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz" +
"abcdefghijklmnopqrstuvwxyz\n"
};
var obj = JObject.FromObject(temp);
var objString = obj.ToString(Formatting.None);
var buffer = Encoding.UTF8.GetBytes(objString);
fixed(byte *pBuffer = buffer)
{
parser.SetBuffer(pBuffer, buffer.Length);
using (var writer = new BlittableJsonDocumentBuilder(ctx,
BlittableJsonDocumentBuilder.UsageMode.None,
"test", parser, state))
{
writer.ReadObjectDocument();
var x = writer.Read();
writer.FinalizeDocument();
using (var reader = writer.CreateReader())
reader.BlittableValidation();
}
}
}
}
}
public unsafe void Valid_object_read_from_non_zero_offset()
{
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var jsonParserState = new JsonParserState();
using (var parser = new UnmanagedJsonParser(context, jsonParserState, "changes/1"))
{
byte[] buffer = new byte[4096];
var bufferOffset = 128; //non-zero offset
var allTokens = new AllTokensTypes
{
Bool = true,
Float = 123.4567F,
Int = 45679123,
IntArray = new[] { 1, 2, 3 },
Null = null,
Object = new Empty(),
String = "qwertyuio"
};
var obj = JObject.FromObject(allTokens);
var objString = obj.ToString(Formatting.None);
var data = Encoding.UTF8.GetBytes(objString);
data.CopyTo(buffer, bufferOffset);
fixed(byte *pBuffer = buffer)
{
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "order/1", parser, jsonParserState))
{
parser.SetBuffer(pBuffer + bufferOffset, data.Length);
builder.ReadObjectDocument();
Assert.True(builder.Read());
builder.FinalizeDocument();
using (var reader = builder.CreateReader())
{
var value = reader.ToString();
Assert.NotNull(value);
}
}
}
}
}
}
private static unsafe BlittableJsonReaderObject ParseJsonAllAtOnce(string s, JsonOperationContext ctx)
{
var jsonParserState = new JsonParserState();
var parser = new UnmanagedJsonParser(ctx, jsonParserState, "test");
var builder = new BlittableJsonDocumentBuilder(ctx, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "test", parser, jsonParserState);
builder.ReadObjectDocument();
var value = ctx.GetLazyString(s);
parser.SetBuffer(value.Buffer, value.Size);
Assert.True(builder.Read());
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
public void FailsOnInvalidJson(string invalidJson)
{
using (var ctx = JsonOperationContext.ShortTermSingleUse())
{
var buffer = Encoding.UTF8.GetBytes(invalidJson);
var state = new JsonParserState();
fixed(byte *pBuffer = buffer)
{
using (var parser = new UnmanagedJsonParser(ctx, state, "test"))
{
parser.SetBuffer(pBuffer, buffer.Length);
var writer = new BlittableJsonDocumentBuilder(ctx, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "test", parser, state);
writer.ReadObjectDocument();
Assert.Throws <InvalidDataException>(() => writer.Read());
}
}
}
}
public static BlittableJsonReaderObject ParseToMemory(
this JsonOperationContext.SyncJsonOperationContext syncContext,
Stream stream,
string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode,
JsonOperationContext.MemoryBuffer bytes,
IBlittableDocumentModifier modifier = null)
{
syncContext.EnsureNotDisposed();
syncContext.JsonParserState.Reset();
using (var parser = new UnmanagedJsonParser(syncContext.Context, syncContext.JsonParserState, debugTag))
using (var builder = new BlittableJsonDocumentBuilder(syncContext.Context, mode, debugTag, parser, syncContext.JsonParserState, modifier: modifier))
{
syncContext.Context.CachedProperties.NewDocument();
builder.ReadObjectDocument();
while (true)
{
if (bytes.Valid == bytes.Used)
{
var read = stream.Read(bytes.Memory.Memory.Span);
syncContext.EnsureNotDisposed();
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
bytes.Valid = read;
bytes.Used = 0;
}
parser.SetBuffer(bytes);
var result = builder.Read();
bytes.Used += parser.BufferOffset;
if (result)
{
break;
}
}
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
}
public BlittableJsonReaderObject ParseToMemory(Stream stream, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode,
ManagedPinnedBuffer bytes, IBlittableDocumentModifier modifier = null)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag))
using (var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState, modifier: modifier))
{
CachedProperties.NewDocument();
builder.ReadObjectDocument();
while (true)
{
if (bytes.Valid == bytes.Used)
{
var read = stream.Read(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
EnsureNotDisposed();
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
bytes.Valid = read;
bytes.Used = 0;
}
parser.SetBuffer(bytes);
var result = builder.Read();
bytes.Used += parser.BufferOffset;
if (result)
{
break;
}
}
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
}
public void CanReadAll(string name)
{
using (var ctx = JsonOperationContext.ShortTermSingleUse())
using (var stream = typeof(UnmanageJsonReaderTests).GetTypeInfo().Assembly.GetManifestResourceStream(name))
using (var parser = new UnmanagedJsonParser(ctx, new JsonParserState(), "test"))
{
var buffer = new byte[4096];
fixed(byte *pBuffer = buffer)
{
while (stream.Position != stream.Length)
{
var read = stream.Read(buffer, 0, buffer.Length);
parser.SetBuffer(pBuffer, read);
while (parser.Read())
{
}
}
}
}
}
public unsafe BlittableJsonReaderArray ParseBufferToArray(string value, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode, IBlittableDocumentModifier modifier = null)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag))
using (var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState, modifier: modifier))
using (GetManagedBuffer(out var buffer))
{
CachedProperties.NewDocument();
builder.ReadArrayDocument();
var maxChars = buffer.Length / 8; //utf8 max size is 8 bytes, must consider worst case possiable
bool lastReadResult = false;
for (int i = 0; i < value.Length; i += maxChars)
{
var charsToRead = Math.Min(value.Length - i, maxChars);
var length = Encodings.Utf8.GetBytes(value, i,
charsToRead,
buffer.Buffer.Array,
buffer.Buffer.Offset);
parser.SetBuffer(buffer.Pointer, length);
lastReadResult = builder.Read();
}
if (lastReadResult == false)
{
throw new EndOfStreamException("Buffer ended without reaching end of json content");
}
builder.FinalizeDocument();
var reader = builder.CreateArrayReader(false);
return(reader);
}
}
private async Task <BlittableJsonReaderObject> ParseToMemoryAsync(Stream stream, string documentId, BlittableJsonDocumentBuilder.UsageMode mode)
{
_jsonParserState.Reset();
ManagedPinnedBuffer bytes;
using (GetManagedBuffer(out bytes))
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, documentId))
{
var writer = new BlittableJsonDocumentBuilder(this, mode, documentId, parser, _jsonParserState);
try
{
CachedProperties.NewDocument();
writer.ReadObjectDocument();
while (true)
{
var read = await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(bytes, read);
if (writer.Read())
{
break;
}
}
writer.FinalizeDocument();
var reader = writer.CreateReader();
RegisterLiveReader(reader);
return(reader);
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
public async Task <BlittableJsonReaderArray> ParseArrayToMemoryAsync(Stream stream, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode)
{
_jsonParserState.Reset();
ManagedPinnedBuffer bytes;
using (GetManagedBuffer(out bytes))
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag))
{
var writer = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState);
try
{
CachedProperties.NewDocument();
writer.ReadArrayDocument();
while (true)
{
var read = await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(bytes, read);
if (writer.Read())
{
break;
}
}
writer.FinalizeDocument();
// here we "leak" the memory used by the array, in practice this is used
// in short scoped context, so we don't care
return(writer.CreateArrayReader());
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
public async Task <BlittableJsonReaderArray> ParseArrayToMemoryAsync(Stream stream, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode)
{
var state = new JsonParserState();
var buffer = GetParsingBuffer();
using (var parser = new UnmanagedJsonParser(this, state, debugTag))
{
var writer = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, state);
try
{
CachedProperties.NewDocument();
writer.ReadArray();
while (true)
{
var read = await stream.ReadAsync(buffer, 0, buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(buffer, read);
if (writer.Read())
{
break;
}
}
writer.FinalizeDocument();
_disposables.Add(writer);
return(writer.CreateArrayReader());
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
private BlittableJsonReaderObject ParseToMemory(Stream stream, string debugTag, BlittableJsonDocumentBuilder.UsageMode mode)
{
var state = new JsonParserState();
var buffer = GetParsingBuffer();
using (var parser = new UnmanagedJsonParser(this, state, debugTag))
{
var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, state);
try
{
CachedProperties.NewDocument();
builder.ReadObject();
while (true)
{
var read = stream.Read(buffer, 0, buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(buffer, read);
if (builder.Read())
{
break;
}
}
builder.FinalizeDocument();
_disposables.Add(builder);
return(builder.CreateReader());
}
catch (Exception)
{
builder.Dispose();
throw;
}
}
}
public async Task <BlittableJsonReaderObject> ReadFromWebSocket(
WebSocket webSocket,
string debugTag,
CancellationToken cancellationToken)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
UnmanagedJsonParser parser = null;
BlittableJsonDocumentBuilder builder = null;
var managedBuffer = default(ReturnBuffer);
var generation = _generation;
try
{
parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag);
builder = new BlittableJsonDocumentBuilder(this,
BlittableJsonDocumentBuilder.UsageMode.None, debugTag, parser, _jsonParserState);
managedBuffer = GetManagedBuffer(out var bytes);
try
{
builder.ReadObjectDocument();
var result = await webSocket.ReceiveAsync(bytes.Buffer, cancellationToken);
EnsureNotDisposed();
if (result.MessageType == WebSocketMessageType.Close)
{
return(null);
}
bytes.Valid = result.Count;
bytes.Used = 0;
parser.SetBuffer(bytes);
while (true)
{
var read = builder.Read();
bytes.Used += parser.BufferOffset;
if (read)
{
break;
}
result = await webSocket.ReceiveAsync(bytes.Buffer, cancellationToken);
bytes.Valid = result.Count;
bytes.Used = 0;
parser.SetBuffer(bytes);
}
builder.FinalizeDocument();
return(builder.CreateReader());
}
catch (Exception)
{
builder.Dispose();
throw;
}
}
finally
{
DisposeIfNeeded(generation, parser, builder);
if (generation == _generation)
{
managedBuffer.Dispose();
}
}
}
private async Task ListenForCommands()
{
await _clusterDashboardNotifications.EnsureWatcher(); // in current impl cluster dashboard senders talk to a single watcher
using (_readContext.GetMemoryBuffer(out JsonOperationContext.MemoryBuffer segment1))
using (_readContext.GetMemoryBuffer(out JsonOperationContext.MemoryBuffer segment2))
{
try
{
var segments = new[] { segment1, segment2 };
int index = 0;
var receiveAsync = _webSocket.ReceiveAsync(segments[index].Memory.Memory, _resourceShutdown);
var jsonParserState = new JsonParserState();
using (var parser = new UnmanagedJsonParser(_readContext, jsonParserState, "cluster-dashboard"))
{
var result = await receiveAsync;
_resourceShutdown.ThrowIfCancellationRequested();
parser.SetBuffer(segments[index], 0, result.Count);
index++;
receiveAsync = _webSocket.ReceiveAsync(segments[index].Memory.Memory, _resourceShutdown);
while (true)
{
using (var builder = new BlittableJsonDocumentBuilder(_readContext, BlittableJsonDocumentBuilder.UsageMode.None,
"cluster-dashboard", parser, jsonParserState))
{
parser.NewDocument();
builder.ReadObjectDocument();
while (builder.Read() == false)
{
result = await receiveAsync;
_resourceShutdown.ThrowIfCancellationRequested();
parser.SetBuffer(segments[index], 0, result.Count);
if (++index >= segments.Length)
{
index = 0;
}
receiveAsync = _webSocket.ReceiveAsync(segments[index].Memory.Memory, _resourceShutdown);
}
builder.FinalizeDocument();
using (var reader = builder.CreateReader())
{
await HandleCommand(reader);
}
}
}
}
}
catch (IOException ex)
{
/* Client was disconnected, write to log */
if (Logger.IsInfoEnabled)
{
Logger.Info("Client was disconnected", ex);
}
}
catch (Exception ex)
{
// if we received close from the client, we want to ignore it and close the websocket (dispose does it)
if (ex is WebSocketException webSocketException &&
webSocketException.WebSocketErrorCode == WebSocketError.InvalidState &&
_webSocket.State == WebSocketState.CloseReceived)
{
// ignore
}
public async Task <ImportResult> Import(DocumentsOperationContext context, Stream stream, Action <IOperationProgress> onProgress = null)
{
var result = new ImportResult();
var progress = new IndeterminateProgress();
var state = new JsonParserState();
JsonOperationContext.ManagedPinnedBuffer buffer;
using (context.GetManagedBuffer(out buffer))
using (var parser = new UnmanagedJsonParser(context, state, "fileName"))
{
var operateOnType = "__top_start_object";
var buildVersion = 0L;
var identities = new Dictionary <string, long>();
VersioningStorage versioningStorage = null;
while (true)
{
if (parser.Read() == false)
{
var read = await stream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length);
if (read == 0)
{
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
break;
}
parser.SetBuffer(buffer, read);
continue;
}
switch (state.CurrentTokenType)
{
case JsonParserToken.String:
unsafe
{
operateOnType =
new LazyStringValue(null, state.StringBuffer, state.StringSize, context).ToString();
}
break;
case JsonParserToken.Integer:
switch (operateOnType)
{
case "BuildVersion":
buildVersion = state.Long;
break;
}
break;
case JsonParserToken.StartObject:
if (operateOnType == "__top_start_object")
{
operateOnType = null;
break;
}
context.CachedProperties.NewDocument();
var builder = new BlittableJsonDocumentBuilder(_batchPutCommand.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "ImportObject", parser, state);
builder.ReadNestedObject();
while (builder.Read() == false)
{
var read = await stream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(buffer, read);
}
builder.FinalizeDocument();
if (operateOnType == "Docs" && Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents))
{
progress.Progress = "Importing Documents";
onProgress?.Invoke(progress);
PatchDocument patch = null;
PatchRequest patchRequest = null;
if (string.IsNullOrWhiteSpace(Options.TransformScript) == false)
{
patch = new PatchDocument(context.DocumentDatabase);
patchRequest = new PatchRequest
{
Script = Options.TransformScript
};
}
result.DocumentsCount++;
var reader = builder.CreateReader();
var document = new Document
{
Data = reader,
};
if (Options.IncludeExpired == false && document.Expired(_database.Time.GetUtcNow()))
{
continue;
}
TransformScriptOrDisableVersioningIfNeeded(context, patch, reader, document,
patchRequest);
_batchPutCommand.Add(document.Data);
if (result.DocumentsCount % 1000 == 0)
{
progress.Progress = $"Imported {result.DocumentsCount} documents";
onProgress?.Invoke(progress);
}
await HandleBatchOfDocuments(context, parser, buildVersion).ConfigureAwait(false);
}
else if (operateOnType == "RevisionDocuments" &&
Options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments))
{
if (versioningStorage == null)
{
break;
}
result.RevisionDocumentsCount++;
var reader = builder.CreateReader();
_batchPutCommand.Add(reader);
await HandleBatchOfDocuments(context, parser, buildVersion).ConfigureAwait(false);;
}
else
{
using (builder)
{
switch (operateOnType)
{
case "Attachments":
result.Warnings.Add("Attachments are not supported anymore. Use RavenFS isntead. Skipping.");
break;
case "Indexes":
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Indexes) == false)
{
continue;
}
result.IndexesCount++;
progress.Progress = "importing Indexes";
onProgress?.Invoke(progress);
try
{
IndexProcessor.Import(builder, _database, buildVersion, Options.RemoveAnalyzers);
}
catch (Exception e)
{
result.Warnings.Add($"Could not import index. Message: {e.Message}");
}
break;
case "Transformers":
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Transformers) == false)
{
continue;
}
result.TransformersCount++;
progress.Progress = "Importing Transformers";
onProgress?.Invoke(progress);
try
{
TransformerProcessor.Import(builder, _database, buildVersion);
}
catch (Exception e)
{
result.Warnings.Add($"Could not import transformer. Message: {e.Message}");
}
break;
case "Identities":
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Identities))
{
result.IdentitiesCount++;
progress.Progress = "Importing Identities";
onProgress?.Invoke(progress);
using (var reader = builder.CreateReader())
{
try
{
string identityKey, identityValueString;
long identityValue;
if (reader.TryGet("Key", out identityKey) == false || reader.TryGet("Value", out identityValueString) == false || long.TryParse(identityValueString, out identityValue) == false)
{
result.Warnings.Add($"Cannot import the following identity: '{reader}'. Skipping.");
}
else
{
identities[identityKey] = identityValue;
}
}
catch (Exception e)
{
result.Warnings.Add($"Cannot import the following identity: '{reader}'. Error: {e}. Skipping.");
}
}
}
break;
default:
result.Warnings.Add(
$"The following type is not recognized: '{operateOnType}'. Skipping.");
break;
}
}
}
break;
case JsonParserToken.StartArray:
switch (operateOnType)
{
case "RevisionDocuments":
// We are taking a reference here since the documents import can activate or disable the versioning.
// We hold a local copy because the user can disable the bundle during the import process, exteranly.
// In this case we want to continue to import the revisions documents.
versioningStorage = _database.BundleLoader.VersioningStorage;
_batchPutCommand.IsRevision = true;
break;
}
break;
case JsonParserToken.EndArray:
switch (operateOnType)
{
case "Docs":
await FinishBatchOfDocuments();
_batchPutCommand = new MergedBatchPutCommand(_database, buildVersion);
break;
case "RevisionDocuments":
await FinishBatchOfDocuments();
break;
case "Identities":
if (identities.Count > 0)
{
using (var tx = context.OpenWriteTransaction())
{
_database.DocumentsStorage.UpdateIdentities(context, identities);
tx.Commit();
}
}
identities = null;
break;
}
break;
}
}
}
return(result);
}
private async Task HandleConnection(WebSocket webSocket, JsonOperationContext context)
{
// this flag can be used to detect if server was restarted between changes connections on client side
var sendStartTime = GetBoolValueQueryString("sendServerStartTime", false).GetValueOrDefault(false);
var throttleConnection = GetBoolValueQueryString("throttleConnection", false).GetValueOrDefault(false);
var connection = new NotificationsClientConnection(webSocket, Database);
Database.Notifications.Connect(connection);
var sendTask = connection.StartSendingNotifications(sendStartTime, throttleConnection);
var debugTag = "changes/" + connection.Id;
JsonOperationContext.ManagedPinnedBuffer segment1, segment2;
using (context.GetManagedBuffer(out segment1))
using (context.GetManagedBuffer(out segment2))
{
try
{
var segments = new[] { segment1, segment2 };
int index = 0;
var receiveAsync = webSocket.ReceiveAsync(segments[index].Buffer, Database.DatabaseShutdown);
var jsonParserState = new JsonParserState();
using (var parser = new UnmanagedJsonParser(context, jsonParserState, debugTag))
{
var result = await receiveAsync;
parser.SetBuffer(segments[index], result.Count);
index++;
receiveAsync = webSocket.ReceiveAsync(segments[index].Buffer, Database.DatabaseShutdown);
while (true)
{
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, debugTag, parser, jsonParserState))
{
parser.NewDocument();
builder.ReadObjectDocument();
while (builder.Read() == false)
{
result = await receiveAsync;
parser.SetBuffer(segments[index], result.Count);
if (++index >= segments.Length)
{
index = 0;
}
receiveAsync = webSocket.ReceiveAsync(segments[index].Buffer, Database.DatabaseShutdown);
}
builder.FinalizeDocument();
using (var reader = builder.CreateReader())
{
string command, commandParameter;
if (reader.TryGet("Command", out command) == false)
{
throw new ArgumentNullException(nameof(command), "Command argument is mandatory");
}
reader.TryGet("Param", out commandParameter);
connection.HandleCommand(command, commandParameter);
int commandId;
if (reader.TryGet("CommandId", out commandId))
{
connection.Confirm(commandId);
}
}
}
}
}
}
catch (IOException ex)
{
/* Client was disconnected, write to log */
if (Logger.IsInfoEnabled)
{
Logger.Info("Client was disconnected", ex);
}
}
finally
{
Database.Notifications.Disconnect(connection.Id);
}
}
await sendTask;
}
private async Task HandleConnection(WebSocket webSocket, JsonOperationContext context)
{
var fromStudio = GetBoolValueQueryString(StudioMarker, false) ?? false;
var throttleConnection = GetBoolValueQueryString("throttleConnection", false).GetValueOrDefault(false);
var connection = new ChangesClientConnection(webSocket, Database, fromStudio);
Database.Changes.Connect(connection);
var sendTask = connection.StartSendingNotifications(throttleConnection);
var debugTag = "changes/" + connection.Id;
using (context.GetManagedBuffer(out JsonOperationContext.ManagedPinnedBuffer segment1))
using (context.GetManagedBuffer(out JsonOperationContext.ManagedPinnedBuffer segment2))
{
try
{
var segments = new[] { segment1, segment2 };
int index = 0;
var receiveAsync = webSocket.ReceiveAsync(segments[index].Buffer, Database.DatabaseShutdown);
var jsonParserState = new JsonParserState();
using (var parser = new UnmanagedJsonParser(context, jsonParserState, debugTag))
{
var result = await receiveAsync;
Database.DatabaseShutdown.ThrowIfCancellationRequested();
parser.SetBuffer(segments[index], 0, result.Count);
index++;
receiveAsync = webSocket.ReceiveAsync(segments[index].Buffer, Database.DatabaseShutdown);
while (true)
{
using (var builder =
new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, debugTag, parser, jsonParserState))
{
parser.NewDocument();
builder.ReadObjectDocument();
while (builder.Read() == false)
{
result = await receiveAsync;
Database.DatabaseShutdown.ThrowIfCancellationRequested();
parser.SetBuffer(segments[index], 0, result.Count);
if (++index >= segments.Length)
{
index = 0;
}
receiveAsync = webSocket.ReceiveAsync(segments[index].Buffer, Database.DatabaseShutdown);
}
builder.FinalizeDocument();
using (var reader = builder.CreateReader())
{
if (reader.TryGet("Command", out string command) == false)
{
throw new ArgumentNullException(nameof(command), "Command argument is mandatory");
}
reader.TryGet("Param", out string commandParameter);
reader.TryGet("Params", out BlittableJsonReaderArray commandParameters);
connection.HandleCommand(command, commandParameter, commandParameters);
if (reader.TryGet("CommandId", out int commandId))
{
connection.Confirm(commandId);
}
}
}
}
}
}
catch (IOException ex)
{
/* Client was disconnected, write to log */
if (Logger.IsInfoEnabled)
{
Logger.Info("Client was disconnected", ex);
}
}
catch
{
#pragma warning disable 4014
sendTask.IgnoreUnobservedExceptions();
#pragma warning restore 4014
throw;
}
finally
{
Database.Changes.Disconnect(connection.Id);
}
}
await sendTask;
}
private unsafe void SetBuffer(UnmanagedJsonParser parser, LazyStringValue value)
{
parser.SetBuffer(value.Buffer, value.Size);
}
public async ValueTask <BlittableJsonReaderObject> ParseToMemoryAsync(Stream stream, string documentId, BlittableJsonDocumentBuilder.UsageMode mode, ManagedPinnedBuffer bytes,
CancellationToken?token = null,
int maxSize = int.MaxValue)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
UnmanagedJsonParser parser = null;
BlittableJsonDocumentBuilder builder = null;
var generation = _generation;
var streamDisposer = token?.Register(stream.Dispose);
try
{
parser = new UnmanagedJsonParser(this, _jsonParserState, documentId);
builder = new BlittableJsonDocumentBuilder(this, mode, documentId, parser, _jsonParserState);
CachedProperties.NewDocument();
builder.ReadObjectDocument();
while (true)
{
token?.ThrowIfCancellationRequested();
if (bytes.Valid == bytes.Used)
{
var read = token.HasValue
? await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length, token.Value)
: await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
EnsureNotDisposed();
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
bytes.Valid = read;
bytes.Used = 0;
maxSize -= read;
if (maxSize < 0)
{
throw new ArgumentException($"The maximum size allowed for {documentId} ({maxSize}) has been exceeded, aborting");
}
}
parser.SetBuffer(bytes);
var result = builder.Read();
bytes.Used += parser.BufferOffset;
if (result)
{
break;
}
}
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
finally
{
streamDisposer?.Dispose();
DisposeIfNeeded(generation, parser, builder);
}
}
private void WriteToServer(string url, Stream serverStream)
{
const string debugTag = "bulk/insert/document";
var jsonParserState = new JsonParserState();
//var streamNetworkBuffer = new BufferedStream(serverStream, 32 * 1024);
var streamNetworkBuffer = serverStream;
var writeToStreamBuffer = new byte[32 * 1024];
var header = Encoding.UTF8.GetBytes(RavenJObject.FromObject(new TcpConnectionHeaderMessage
{
DatabaseName = MultiDatabase.GetDatabaseName(url),
Operation = TcpConnectionHeaderMessage.OperationTypes.BulkInsert
}).ToString());
streamNetworkBuffer.Write(header, 0, header.Length);
JsonOperationContext.ManagedPinnedBuffer bytes;
using (_jsonOperationContext.GetManagedBuffer(out bytes))
{
while (_documents.IsCompleted == false)
{
_cts.Token.ThrowIfCancellationRequested();
MemoryStream jsonBuffer;
try
{
jsonBuffer = _documents.Take();
}
catch (InvalidOperationException)
{
break;
}
var needToThrottle = _throttlingEvent.Wait(0) == false;
_jsonOperationContext.ResetAndRenew();
using (var jsonParser = new UnmanagedJsonParser(_jsonOperationContext, jsonParserState, debugTag))
using (var builder = new BlittableJsonDocumentBuilder(_jsonOperationContext,
BlittableJsonDocumentBuilder.UsageMode.ToDisk, debugTag,
jsonParser, jsonParserState))
{
_jsonOperationContext.CachedProperties.NewDocument();
builder.ReadObjectDocument();
while (true)
{
var read = jsonBuffer.Read(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
jsonParser.SetBuffer(bytes, read);
if (builder.Read())
{
break;
}
}
_buffers.Add(jsonBuffer);
builder.FinalizeDocument();
WriteVariableSizeInt(streamNetworkBuffer, builder.SizeInBytes);
WriteToStream(streamNetworkBuffer, builder, writeToStreamBuffer);
}
if (needToThrottle)
{
streamNetworkBuffer.Flush();
_throttlingEvent.Wait(500);
}
}
streamNetworkBuffer.WriteByte(0); //done
streamNetworkBuffer.Flush();
}
}
private async Task <BlittableJsonReaderObject> Receive(ClientWebSocket webSocket,
JsonOperationContext context)
{
BlittableJsonDocumentBuilder builder = null;
try
{
if (webSocket.State != WebSocketState.Open)
{
throw new InvalidOperationException(
$"Trying to 'ReceiveAsync' WebSocket while not in Open state. State is {webSocket.State}");
}
var state = new JsonParserState();
JsonOperationContext.ManagedPinnedBuffer buffer;
using (context.GetManagedBuffer(out buffer))
using (var parser = new UnmanagedJsonParser(context, state, "")) //TODO: FIXME
{
builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None,
nameof(TrafficRec) + "." + nameof(Receive), parser, state);
builder.ReadObjectDocument();
while (builder.Read() == false)
{
var result = await webSocket.ReceiveAsync(buffer.Buffer, CancellationToken.None);
if (result.MessageType == WebSocketMessageType.Close)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Client got close message from server and is closing connection");
}
builder.Dispose();
// actual socket close from dispose
return(null);
}
if (result.EndOfMessage == false)
{
throw new EndOfStreamException("Stream ended without reaching end of json content.");
}
parser.SetBuffer(buffer, 0, result.Count);
}
builder.FinalizeDocument();
return(builder.CreateReader());
}
}
catch (WebSocketException ex)
{
builder?.Dispose();
if (_logger.IsInfoEnabled)
{
_logger.Info("Failed to receive a message, client was probably disconnected", ex);
}
throw;
}
}
public async Task <bool> MoveNextAsync()
{
var state = new JsonParserState();
JsonOperationContext.ManagedPinnedBuffer buffer;
using (var parser = new UnmanagedJsonParser(_session.Context, state, "stream contents"))
using (_session.Context.GetManagedBuffer(out buffer))
{
await ReadNextTokenAsync(_response.Stream, parser, buffer).ConfigureAwait(false);
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
throw new InvalidOperationException("Expected stream to start, but got " +
state.CurrentTokenType);
}
await ReadNextTokenAsync(_response.Stream, parser, buffer).ConfigureAwait(false);
if (state.CurrentTokenType != JsonParserToken.String)
{
throw new InvalidOperationException("Expected stream intial property, but got " +
state.CurrentTokenType);
}
// TODO: Need to handle initial properties here from QueryHeaderInformation
var propery = GetPropertyName(state);
if (propery.Equals("Results") == false)
{
throw new InvalidOperationException("Expected stream property 'Results' but got " + propery);
}
await ReadNextTokenAsync(_response.Stream, parser, buffer).ConfigureAwait(false);
if (state.CurrentTokenType != JsonParserToken.StartArray)
{
throw new InvalidOperationException("Expected stream intial property, but got " +
state.CurrentTokenType);
}
await ReadNextTokenAsync(_response.Stream, parser, buffer).ConfigureAwait(false);
_session.Context.CachedProperties.NewDocument();
var builder = new BlittableJsonDocumentBuilder(_session.Context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, "ImportObject", parser, state);
builder.ReadNestedObject();
while (builder.Read() == false)
{
var read = await _response.Stream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length).ConfigureAwait(false);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(buffer, read);
}
builder.FinalizeDocument();
await ReadNextTokenAsync(_response.Stream, parser, buffer).ConfigureAwait(false);
Current = builder.CreateReader();
if (state.CurrentTokenType == JsonParserToken.EndArray)
{
await ReadNextTokenAsync(_response.Stream, parser, buffer).ConfigureAwait(false);
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
throw new InvalidOperationException("Expected stream closing token, but got " +
state.CurrentTokenType);
}
return(false);
}
await ReadNextTokenAsync(_response.Stream, parser, buffer).ConfigureAwait(false);
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
throw new InvalidOperationException("Expected stream closing token, but got " +
state.CurrentTokenType);
}
return(true);
}
}