private static async Task <BlittableJsonReaderArray> ReadJsonArray(
JsonOperationContext ctx,
Stream stream,
UnmanagedJsonParser parser,
JsonParserState state,
JsonOperationContext.ManagedPinnedBuffer buffer,
CancellationToken token)
{
BlittableJsonReaderArray reader;
using (var builder = new BlittableJsonDocumentBuilder(
ctx,
BlittableJsonDocumentBuilder.UsageMode.ToDisk,
"json/array", parser, state))
{
ctx.CachedProperties.NewDocument();
builder.ReadArrayDocument();
while (true)
{
if (builder.Read())
{
break;
}
await RefillParserBuffer(stream, buffer, parser, token);
}
builder.FinalizeDocument();
reader = builder.CreateArrayReader(noCache: true);
}
return(reader);
}
public RemoteConnection(string src, Stream stream, long term, Action disconnect, [CallerMemberName] string caller = null)
{
_log = LoggingSource.Instance.GetLogger <RemoteConnection>($"{src} > [? discovering... ? ]");
_stream = stream;
_disconnect = disconnect;
_buffer = JsonOperationContext.ManagedPinnedBuffer.LongLivedInstance();
RegisterConnection("?", term, caller);
}
public RemoteConnection(string dest, string src, Stream stream)
{
_destTag = dest;
_src = src;
_log = LoggingSource.Instance.GetLogger <RemoteConnection>($"{_src} > {_destTag}");
_stream = stream;
_buffer = JsonOperationContext.ManagedPinnedBuffer.LongLivedInstance();
}
private static unsafe void PartialComputeHash(JsonOperationContext.ManagedPinnedBuffer cryptoState, JsonOperationContext.ManagedPinnedBuffer buffer, int bufferRead)
{
var rc = Sodium.crypto_generichash_update(cryptoState.Pointer, buffer.Pointer, (ulong)bufferRead);
if (rc != 0)
{
throw new InvalidOperationException("Unable to hash attachment: " + rc);
}
}
public ReadMany(JsonOperationContext ctx, Stream stream, JsonOperationContext.ManagedPinnedBuffer buffer, CancellationToken token)
{
_stream = stream;
_buffer = buffer;
_token = token;
_state = new JsonParserState();
_parser = new UnmanagedJsonParser(ctx, _state, "bulk_docs");
}
private static unsafe void InitComputeHash(JsonOperationContext.ManagedPinnedBuffer cryptoState)
{
var rc = Sodium.crypto_generichash_init(cryptoState.Pointer, null, UIntPtr.Zero, Sodium.crypto_generichash_bytes());
if (rc != 0)
{
throw new InvalidOperationException("Unable to hash attachment: " + rc);
}
}
public RemoteConnection(string dest, string src, long term, Stream stream, Action disconnect, [CallerMemberName] string caller = null)
{
_destTag = dest;
_src = src;
_disconnect = disconnect;
_log = LoggingSource.Instance.GetLogger <RemoteConnection>($"{_src} > {_destTag}");
_stream = stream;
_buffer = JsonOperationContext.ManagedPinnedBuffer.LongLivedInstance();
RegisterConnection(dest, term, caller);
}
private static unsafe string FinalizeGetHash(JsonOperationContext.ManagedPinnedBuffer cryptoState, JsonOperationContext.ManagedPinnedBuffer buffer)
{
var size = Sodium.crypto_generichash_bytes();
var rc = Sodium.crypto_generichash_final(cryptoState.Pointer, buffer.Pointer, size);
if (rc != 0)
{
throw new InvalidOperationException("Unable to hash attachment: " + rc);
}
return(Convert.ToBase64String(buffer.Buffer.Array, buffer.Buffer.Offset, (int)size));
}
private static void ReadNextToken(Stream stream, UnmanagedJsonParser parser, JsonOperationContext.ManagedPinnedBuffer buffer)
{
while (parser.Read() == false)
{
var read = stream.Read(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Buffer.Count);
if (read == 0)
{
throw new EndOfStreamException("The stream ended unexpectedly");
}
parser.SetBuffer(buffer, read);
}
}
private static async Task RefillParserBuffer(Stream stream, JsonOperationContext.ManagedPinnedBuffer buffer, UnmanagedJsonParser parser, CancellationToken token = default)
{
// Although we using here WithCancellation and passing the token,
// the stream will stay open even after the cancellation until the entire server will be disposed.
var read = await stream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Buffer.Count, token).WithCancellation(token);
if (read == 0)
{
ThrowUnexpectedEndOfStream();
}
parser.SetBuffer(buffer, 0, read);
}
public Result ParseToMemory(
AsyncManualResetEvent interrupt,
string debugTag,
int timeout,
JsonOperationContext.ManagedPinnedBuffer buffer,
CancellationToken token)
{
if (_prevCall == null)
{
_prevCall = ReadNextObject(debugTag, buffer, token);
_previousWait.Clear();
}
if (_prevCall.IsCompleted)
{
return(ReturnAndClearValue());
}
if (_previousWait.TryGetValue(interrupt, out Task <Task> task) == false)
{
_previousWait[interrupt] = task = Task.WhenAny(_prevCall, interrupt.WaitAsync());
}
if (task.Wait(timeout, token) == false)
{
return new Result {
Timeout = true
}
}
;
if (task.Result != _prevCall)
{
_previousWait.Remove(interrupt);
return(new Result {
Interrupted = true
});
}
return(ReturnAndClearValue());
}
private async Task <Result> ReadNextObject(string debugTag, JsonOperationContext.ManagedPinnedBuffer buffer, CancellationToken token)
{
var returnContext = _contextPool.AllocateOperationContext(out DocumentsOperationContext context);
try
{
var json = await context.ParseToMemoryAsync(_stream, debugTag, BlittableJsonDocumentBuilder.UsageMode.None, buffer, token);
return(new Result
{
Document = json,
ReturnContext = returnContext,
Context = context
});
}
catch (Exception)
{
returnContext.Dispose();
throw;
}
}
public void Dispose()
{
if (_isDisposed)
{
return;
}
Stream?.Dispose();
TcpClient?.Dispose();
_running.Wait();
try
{
if (_isDisposed)
{
return;
}
_isDisposed = true;
MetricsScheduler.Instance.StopTickingMetric(_bytesSentMetric);
MetricsScheduler.Instance.StopTickingMetric(_bytesReceivedMetric);
DocumentDatabase?.RunningTcpConnections.TryRemove(this);
PinnedBuffer?.Dispose();
PinnedBuffer = null;
Stream = null;
TcpClient = null;
}
finally
{
_running.Release();
}
// we'll let the _running be finalized, because otherwise we have
// a possible race condition on dispose
}
private static async Task <CommandData> ReadSingleCommand(
JsonOperationContext ctx,
Stream stream,
JsonParserState state,
UnmanagedJsonParser parser,
JsonOperationContext.ManagedPinnedBuffer buffer,
CancellationToken token)
{
var commandData = new CommandData();
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowUnexpectedToken(JsonParserToken.StartObject, state);
}
while (true)
{
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
if (state.CurrentTokenType == JsonParserToken.EndObject)
{
break;
}
if (state.CurrentTokenType != JsonParserToken.String)
{
ThrowUnexpectedToken(JsonParserToken.String, state);
}
switch (GetPropertyType(state))
{
case CommandPropertyName.Type:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
if (state.CurrentTokenType != JsonParserToken.String)
{
ThrowUnexpectedToken(JsonParserToken.String, state);
}
commandData.Type = GetCommandType(state, ctx);
break;
case CommandPropertyName.Id:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
switch (state.CurrentTokenType)
{
case JsonParserToken.Null:
commandData.Id = null;
break;
case JsonParserToken.String:
commandData.Id = GetStringPropertyValue(state);
break;
default:
ThrowUnexpectedToken(JsonParserToken.String, state);
break;
}
break;
case CommandPropertyName.Name:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
switch (state.CurrentTokenType)
{
case JsonParserToken.Null:
commandData.Name = null;
break;
case JsonParserToken.String:
commandData.Name = GetStringPropertyValue(state);
break;
default:
ThrowUnexpectedToken(JsonParserToken.String, state);
break;
}
break;
case CommandPropertyName.DestinationId:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
switch (state.CurrentTokenType)
{
case JsonParserToken.Null:
commandData.DestinationId = null;
break;
case JsonParserToken.String:
commandData.DestinationId = GetStringPropertyValue(state);
break;
default:
ThrowUnexpectedToken(JsonParserToken.String, state);
break;
}
break;
case CommandPropertyName.DestinationName:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
switch (state.CurrentTokenType)
{
case JsonParserToken.Null:
commandData.DestinationName = null;
break;
case JsonParserToken.String:
commandData.DestinationName = GetStringPropertyValue(state);
break;
default:
ThrowUnexpectedToken(JsonParserToken.String, state);
break;
}
break;
case CommandPropertyName.ContentType:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
switch (state.CurrentTokenType)
{
case JsonParserToken.Null:
commandData.ContentType = string.Empty;
break;
case JsonParserToken.String:
commandData.ContentType = GetStringPropertyValue(state);
break;
default:
ThrowUnexpectedToken(JsonParserToken.String, state);
break;
}
break;
case CommandPropertyName.Document:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
commandData.Document = await ReadJsonObject(ctx, stream, commandData.Id, parser, state, buffer, token);
break;
case CommandPropertyName.Patch:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
var patch = await ReadJsonObject(ctx, stream, commandData.Id, parser, state, buffer, token);
commandData.Patch = PatchRequest.Parse(patch, out commandData.PatchArgs);
break;
case CommandPropertyName.PatchIfMissing:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
var patchIfMissing = await ReadJsonObject(ctx, stream, commandData.Id, parser, state, buffer, token);
commandData.PatchIfMissing = PatchRequest.Parse(patchIfMissing, out commandData.PatchIfMissingArgs);
break;
case CommandPropertyName.ChangeVector:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
if (state.CurrentTokenType == JsonParserToken.Null)
{
commandData.ChangeVector = null;
}
else
{
if (state.CurrentTokenType != JsonParserToken.String)
{
ThrowUnexpectedToken(JsonParserToken.String, state);
}
commandData.ChangeVector = GetLazyStringValue(ctx, state);
}
break;
case CommandPropertyName.Index:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
if (state.CurrentTokenType != JsonParserToken.Integer)
{
ThrowUnexpectedToken(JsonParserToken.True, state);
}
commandData.Index = state.Long;
break;
case CommandPropertyName.IdPrefixed:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
if (state.CurrentTokenType != JsonParserToken.True && state.CurrentTokenType != JsonParserToken.False)
{
ThrowUnexpectedToken(JsonParserToken.True, state);
}
commandData.IdPrefixed = state.CurrentTokenType == JsonParserToken.True;
break;
case CommandPropertyName.ReturnDocument:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
if (state.CurrentTokenType != JsonParserToken.True && state.CurrentTokenType != JsonParserToken.False)
{
ThrowUnexpectedToken(JsonParserToken.True, state);
}
commandData.ReturnDocument = state.CurrentTokenType == JsonParserToken.True;
break;
case CommandPropertyName.Counters:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
var counterOps = await ReadJsonObject(ctx, stream, commandData.Id, parser, state, buffer, token);
commandData.Counters = DocumentCountersOperation.Parse(counterOps);
break;
case CommandPropertyName.FromEtl:
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
if (state.CurrentTokenType != JsonParserToken.True && state.CurrentTokenType != JsonParserToken.False)
{
ThrowUnexpectedToken(JsonParserToken.True, state);
}
commandData.FromEtl = state.CurrentTokenType == JsonParserToken.True;
break;
case CommandPropertyName.NoSuchProperty:
// unknown command - ignore it
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser, token);
}
if (state.CurrentTokenType == JsonParserToken.StartObject ||
state.CurrentTokenType == JsonParserToken.StartArray)
{
await ReadJsonObject(ctx, stream, commandData.Id, parser, state, buffer, token);
}
break;
}
}
switch (commandData.Type)
{
case CommandType.None:
ThrowInvalidType();
break;
case CommandType.PUT:
if (commandData.Document == null)
{
ThrowMissingDocumentProperty();
}
break;
case CommandType.PATCH:
if (commandData.Patch == null)
{
ThrowMissingPatchProperty();
}
break;
case CommandType.AttachmentPUT:
if (commandData.Name == null)
{
ThrowMissingNameProperty();
}
break;
case CommandType.Counters:
if (commandData.Counters == null)
{
ThrowMissingNameProperty();
}
break;
}
return(commandData);
}
private static async Task <BlittableJsonReaderObject> ReadJsonObject(JsonOperationContext ctx, Stream stream, string id, UnmanagedJsonParser parser,
JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer, CancellationToken token)
{
if (state.CurrentTokenType == JsonParserToken.Null)
{
return(null);
}
BlittableJsonReaderObject reader;
using (var builder = new BlittableJsonDocumentBuilder(ctx,
BlittableJsonDocumentBuilder.UsageMode.ToDisk,
id, parser, state, modifier: new BlittableMetadataModifier(ctx)))
{
ctx.CachedProperties.NewDocument();
builder.ReadNestedObject();
while (true)
{
if (builder.Read())
{
break;
}
await RefillParserBuffer(stream, buffer, parser, token);
}
builder.FinalizeDocument();
reader = builder.CreateReader();
reader.NoCache = true;
}
return(reader);
}
private static async Task <bool> IsClusterTransaction(Stream stream, UnmanagedJsonParser parser, JsonOperationContext.ManagedPinnedBuffer buffer, JsonParserState state)
{
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser);
}
if (ReadClusterTransactionProperty(state))
{
while (parser.Read() == false)
{
await RefillParserBuffer(stream, buffer, parser);
}
return(GetStringPropertyValue(state) == nameof(TransactionMode.ClusterWide));
}
return(false);
}
public RemoteConnection(string dest, Stream stream)
{
_destTag = dest;
_stream = stream;
_buffer = JsonOperationContext.ManagedPinnedBuffer.LongLivedInstance();
}
public static void ReadObject(BlittableJsonDocumentBuilder builder, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonOperationContext.ManagedPinnedBuffer buffer)
{
builder.ReadNestedObject();
while (builder.Read() == false)
{
var read = peepingTomStream.Read(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content" + GetPeepingTomBufferAsString(peepingTomStream));
}
parser.SetBuffer(buffer, 0, read);
}
builder.FinalizeDocument();
}
public void SetBuffer(JsonOperationContext.ManagedPinnedBuffer inputBuffer, int offset, int size)
{
SetBuffer(inputBuffer.Pointer + offset, size);
}
public static void SendSubscriptionDocuments(TcpConnectionOptions tcpConnectionOptions, JsonOperationContext.ManagedPinnedBuffer buffer)
{
var remoteEndPoint = tcpConnectionOptions.TcpClient.Client.RemoteEndPoint;
var tcpConnectionDisposable = tcpConnectionOptions.ConnectionProcessingInProgress("Subscription");
try
{
var connection = new SubscriptionConnection(tcpConnectionOptions, tcpConnectionDisposable, buffer);
try
{
Task.Run(async() =>
{
using (tcpConnectionOptions)
using (tcpConnectionDisposable)
using (connection)
{
try
{
bool gotSemaphore;
if ((gotSemaphore = tcpConnectionOptions.DocumentDatabase.SubscriptionStorage.TryEnterSemaphore()) == false)
{
throw new SubscriptionClosedException(
$"Cannot open new subscription connection, max amount of concurrent connections reached ({tcpConnectionOptions.DocumentDatabase.Configuration.Subscriptions.MaxNumberOfConcurrentConnections})");
}
try
{
await connection.InitAsync();
await connection.ProcessSubscriptionAsync();
}
finally
{
if (gotSemaphore)
{
tcpConnectionOptions.DocumentDatabase.SubscriptionStorage.ReleaseSubscriptionsSemaphore();
}
}
}
catch (Exception e)
{
if (connection._logger.IsInfoEnabled)
{
connection._logger.Info(
$"Failed to process subscription {connection.SubscriptionId} / from client {remoteEndPoint}",
e);
}
try
{
await ReportExceptionToClient(connection, connection.ConnectionException ?? e);
}
catch (Exception)
{
// ignored
}
}
finally
{
if (connection._logger.IsInfoEnabled)
{
connection._logger.Info(
$"Finished processing subscription {connection.SubscriptionId} / from client {remoteEndPoint}");
}
}
}
});
}
catch (Exception)
{
connection?.Dispose();
throw;
}
}
catch (Exception)
{
tcpConnectionDisposable?.Dispose();
throw;
}
}
public static unsafe string ReadString(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer)
{
if (Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.String)
{
ThrowInvalidJson(peepingTomStream);
}
return(context.AllocateStringValue(null, state.StringBuffer, state.StringSize).ToString());
}
public void SetBuffer(JsonOperationContext.ManagedPinnedBuffer inputBuffer)
{
SetBuffer(inputBuffer.Pointer + inputBuffer.Used, inputBuffer.Valid - inputBuffer.Used);
}
public static async Task ReadObjectAsync(BlittableJsonDocumentBuilder builder, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonOperationContext.ManagedPinnedBuffer buffer)
{
builder.ReadNestedObject();
while (builder.Read() == false)
{
var read = await peepingTomStream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length).ConfigureAwait(false);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(buffer, 0, read);
}
builder.FinalizeDocument();
}
public static IEnumerable <BlittableJsonReaderObject> ReadArrayToMemory(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer)
{
if (Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.StartArray)
{
ThrowInvalidJson(peepingTomStream);
}
while (true)
{
if (Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "readArray/singleResult", parser, state))
{
ReadObject(builder, peepingTomStream, parser, buffer);
yield return(builder.CreateReader());
}
}
}
public static bool Read(PeepingTomStream stream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer)
{
while (parser.Read() == false)
{
var read = stream.Read(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length);
if (read == 0)
{
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
return(false);
}
parser.SetBuffer(buffer, 0, read);
}
return(true);
}
public void AcceptIncomingConnection(TcpConnectionOptions tcpConnectionOptions, JsonOperationContext.ManagedPinnedBuffer buffer)
{
ReplicationLatestEtagRequest getLatestEtagMessage;
using (tcpConnectionOptions.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var readerObject = context.ParseToMemory(
tcpConnectionOptions.Stream,
"IncomingReplication/get-last-etag-message read",
BlittableJsonDocumentBuilder.UsageMode.None,
buffer))
{
getLatestEtagMessage = JsonDeserializationServer.ReplicationLatestEtagRequest(readerObject);
if (_log.IsInfoEnabled)
{
_log.Info(
$"GetLastEtag: {getLatestEtagMessage.SourceTag}({getLatestEtagMessage.SourceMachineName}) / {getLatestEtagMessage.SourceDatabaseName} ({getLatestEtagMessage.SourceDatabaseId}) - {getLatestEtagMessage.SourceUrl}");
}
}
var connectionInfo = IncomingConnectionInfo.FromGetLatestEtag(getLatestEtagMessage);
try
{
AssertValidConnection(connectionInfo);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Connection from [{connectionInfo}] is rejected.", e);
}
var incomingConnectionRejectionInfos = _incomingRejectionStats.GetOrAdd(connectionInfo,
_ => new ConcurrentQueue <IncomingConnectionRejectionInfo>());
incomingConnectionRejectionInfos.Enqueue(new IncomingConnectionRejectionInfo {
Reason = e.ToString()
});
try
{
tcpConnectionOptions.Dispose();
}
catch
{
// do nothing
}
throw;
}
try
{
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsOperationContext))
using (Database.ConfigurationStorage.ContextPool.AllocateOperationContext(out TransactionOperationContext configurationContext))
using (var writer = new BlittableJsonTextWriter(documentsOperationContext, tcpConnectionOptions.Stream))
using (documentsOperationContext.OpenReadTransaction())
using (configurationContext.OpenReadTransaction())
{
var changeVector = DocumentsStorage.GetDatabaseChangeVector(documentsOperationContext);
var lastEtagFromSrc = DocumentsStorage.GetLastReplicatedEtagFrom(
documentsOperationContext, getLatestEtagMessage.SourceDatabaseId);
if (_log.IsInfoEnabled)
{
_log.Info($"GetLastEtag response, last etag: {lastEtagFromSrc}");
}
var response = new DynamicJsonValue
{
[nameof(ReplicationMessageReply.Type)] = "Ok",
[nameof(ReplicationMessageReply.MessageType)] = ReplicationMessageType.Heartbeat,
[nameof(ReplicationMessageReply.LastEtagAccepted)] = lastEtagFromSrc,
[nameof(ReplicationMessageReply.NodeTag)] = _server.NodeTag,
[nameof(ReplicationMessageReply.DatabaseChangeVector)] = changeVector
};
documentsOperationContext.Write(writer, response);
writer.Flush();
}
}
catch (Exception)
{
try
{
tcpConnectionOptions.Dispose();
}
catch (Exception)
{
// do nothing
}
throw;
}
var newIncoming = new IncomingReplicationHandler(
tcpConnectionOptions,
getLatestEtagMessage,
this,
buffer);
newIncoming.Failed += OnIncomingReceiveFailed;
newIncoming.DocumentsReceived += OnIncomingReceiveSucceeded;
if (_log.IsInfoEnabled)
{
_log.Info(
$"Initialized document replication connection from {connectionInfo.SourceDatabaseName} located at {connectionInfo.SourceUrl}");
}
// need to safeguard against two concurrent connection attempts
var newConnection = _incoming.GetOrAdd(newIncoming.ConnectionInfo.SourceDatabaseId, newIncoming);
if (newConnection == newIncoming)
{
newIncoming.Start();
IncomingReplicationAdded?.Invoke(newIncoming);
ForceTryReconnectAll();
}
else
{
newIncoming.Dispose();
}
}
private void ReplicateToDestination()
{
try
{
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiate);
NativeMemory.EnsureRegistered();
if (_log.IsInfoEnabled)
{
_log.Info($"Will replicate to {Destination.FromString()} via {_connectionInfo.Url}");
}
using (_parent._server.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var record = _parent.LoadDatabaseRecord();
if (record == null)
{
throw new InvalidOperationException($"The database record for {_parent.Database.Name} does not exist?!");
}
if (record.Encrypted && Destination.Url.StartsWith("https:", StringComparison.OrdinalIgnoreCase) == false)
{
throw new InvalidOperationException(
$"{record.DatabaseName} is encrypted, and require HTTPS for replication, but had endpoint with url {Destination.Url} to database {Destination.Database}");
}
}
var task = TcpUtils.ConnectSocketAsync(_connectionInfo, _parent._server.Engine.TcpConnectionTimeout, _log);
task.Wait(CancellationToken);
using (Interlocked.Exchange(ref _tcpClient, task.Result))
{
var wrapSsl = TcpUtils.WrapStreamWithSslAsync(_tcpClient, _connectionInfo, _parent._server.Server.Certificate.Certificate, _parent._server.Engine.TcpConnectionTimeout);
wrapSsl.Wait(CancellationToken);
using (_stream = wrapSsl.Result) // note that _stream is being disposed by the interruptible read
using (_interruptibleRead = new InterruptibleRead(_database.DocumentsStorage.ContextPool, _stream))
using (_buffer = JsonOperationContext.ManagedPinnedBuffer.LongLivedInstance())
{
var documentSender = new ReplicationDocumentSender(_stream, this, _log);
WriteHeaderToRemotePeer();
//handle initial response to last etag and staff
try
{
var response = HandleServerResponse(getFullResponse: true);
switch (response.ReplyType)
{
//The first time we start replication we need to register the destination current CV
case ReplicationMessageReply.ReplyType.Ok:
LastAcceptedChangeVector = response.Reply.DatabaseChangeVector;
break;
case ReplicationMessageReply.ReplyType.Error:
var exception = new InvalidOperationException(response.Reply.Exception);
if (response.Reply.Exception.Contains(nameof(DatabaseDoesNotExistException)) ||
response.Reply.Exception.Contains(nameof(DatabaseNotRelevantException)))
{
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, "Database does not exist");
DatabaseDoesNotExistException.ThrowWithMessageAndException(Destination.Database, response.Reply.Message, exception);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, $"Got error: {response.Reply.Exception}");
throw exception;
}
}
catch (DatabaseDoesNotExistException e)
{
var msg = $"Failed to parse initial server replication response, because there is no database named {_database.Name} " +
"on the other end. ";
if (_external)
{
msg += "In order for the replication to work, a database with the same name needs to be created at the destination";
}
var young = (DateTime.UtcNow - _startedAt).TotalSeconds < 30;
if (young)
{
msg += "This can happen if the other node wasn't yet notified about being assigned this database and should be resolved shortly.";
}
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg);
// won't add an alert on young connections
// because it may take a few seconds for the other side to be notified by
// the cluster that it has this db.
if (young == false)
{
AddAlertOnFailureToReachOtherSide(msg, e);
}
throw;
}
catch (OperationCanceledException e)
{
const string msg = "Got operation canceled notification while opening outgoing replication channel. " +
"Aborting and closing the channel.";
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg);
throw;
}
catch (Exception e)
{
var msg = $"{OutgoingReplicationThreadName} got an unexpected exception during initial handshake";
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg);
AddAlertOnFailureToReachOtherSide(msg, e);
throw;
}
DateTime nextReplicateAt = default(DateTime);
while (_cts.IsCancellationRequested == false)
{
while (_database.Time.GetUtcNow() > nextReplicateAt)
{
if (_parent.DebugWaitAndRunReplicationOnce != null)
{
_parent.DebugWaitAndRunReplicationOnce.Wait(_cts.Token);
_parent.DebugWaitAndRunReplicationOnce.Reset();
}
var sp = Stopwatch.StartNew();
var stats = _lastStats = new OutgoingReplicationStatsAggregator(_parent.GetNextReplicationStatsId(), _lastStats);
AddReplicationPerformance(stats);
AddReplicationPulse(ReplicationPulseDirection.OutgoingBegin);
try
{
using (var scope = stats.CreateScope())
{
try
{
if (Destination is InternalReplication dest)
{
_parent.EnsureNotDeleted(dest.NodeTag);
}
var didWork = documentSender.ExecuteReplicationOnce(scope, ref nextReplicateAt);
if (didWork == false)
{
break;
}
if (Destination is ExternalReplication externalReplication)
{
var taskId = externalReplication.TaskId;
UpdateExternalReplicationInfo(taskId);
}
DocumentsSend?.Invoke(this);
if (sp.ElapsedMilliseconds > 60 * 1000)
{
_waitForChanges.Set();
break;
}
}
catch (OperationCanceledException)
{
// cancellation is not an actual error,
// it is a "notification" that we need to cancel current operation
const string msg = "Operation was canceled.";
AddReplicationPulse(ReplicationPulseDirection.OutgoingError, msg);
throw;
}
catch (Exception e)
{
AddReplicationPulse(ReplicationPulseDirection.OutgoingError, e.Message);
scope.AddError(e);
throw;
}
}
}
finally
{
stats.Complete();
AddReplicationPulse(ReplicationPulseDirection.OutgoingEnd);
}
}
//if this returns false, this means either timeout or canceled token is activated
while (WaitForChanges(_parent.MinimalHeartbeatInterval, _cts.Token) == false)
{
//If we got cancelled we need to break right away
if (_cts.IsCancellationRequested)
{
break;
}
// open tx
// read current change vector compare to last sent
// if okay, send cv
using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (var tx = ctx.OpenReadTransaction())
{
var etag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction);
if (etag == _lastSentDocumentEtag)
{
SendHeartbeat(DocumentsStorage.GetDatabaseChangeVector(ctx));
_parent.CompleteDeletionIfNeeded();
}
else if (nextReplicateAt > DateTime.UtcNow)
{
SendHeartbeat(null);
}
else
{
//Send a heartbeat first so we will get an updated CV of the destination
var currentChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
SendHeartbeat(null);
//If our previous CV is already merged to the destination wait a bit more
if (ChangeVectorUtils.GetConflictStatus(LastAcceptedChangeVector, currentChangeVector) ==
ConflictStatus.AlreadyMerged)
{
continue;
}
// we have updates that we need to send to the other side
// let's do that..
// this can happen if we got replication from another node
// that we need to send to it. Note that we typically
// will wait for the other node to send the data directly to
// our destination, but if it doesn't, we'll step in.
// In this case, we try to limit congestion in the network and
// only send updates that we have gotten from someone else after
// a certain time, to let the other side tell us that it already
// got it. Note that this is merely an optimization to reduce network
// traffic. It is fine to have the same data come from different sources.
break;
}
}
}
_waitForChanges.Reset();
}
}
}
}
catch (AggregateException e)
{
if (e.InnerExceptions.Count == 1)
{
if (e.InnerException is OperationCanceledException oce)
{
HandleOperationCancelException(oce);
}
if (e.InnerException is IOException ioe)
{
HandleIOException(ioe);
}
}
HandleException(e);
}
catch (OperationCanceledException e)
{
HandleOperationCancelException(e);
}
catch (IOException e)
{
HandleIOException(e);
}
catch (Exception e)
{
HandleException(e);
}
void HandleOperationCancelException(OperationCanceledException e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Operation canceled on replication thread ({FromToString}). " +
$"This is not necessary due to an issue. Stopped the thread.");
}
if (_cts.IsCancellationRequested == false)
{
Failed?.Invoke(this, e);
}
}
void HandleIOException(IOException e)
{
if (_log.IsInfoEnabled)
{
if (e.InnerException is SocketException)
{
_log.Info($"SocketException was thrown from the connection to remote node ({FromToString}). " +
$"This might mean that the remote node is done or there is a network issue.", e);
}
else
{
_log.Info($"IOException was thrown from the connection to remote node ({FromToString}).", e);
}
}
Failed?.Invoke(this, e);
}
void HandleException(Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Unexpected exception occurred on replication thread ({FromToString}). " +
$"Replication stopped (will be retried later).", e);
}
Failed?.Invoke(this, e);
}
}
public static long ReadLong(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer)
{
if (Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.Integer)
{
ThrowInvalidJson(peepingTomStream);
}
return(state.Long);
}
public SubscriptionConnection(TcpConnectionOptions connectionOptions, IDisposable tcpConnectionDisposable, JsonOperationContext.ManagedPinnedBuffer bufferToCopy)
{
TcpConnection = connectionOptions;
_tcpConnectionDisposable = tcpConnectionDisposable;
ClientUri = connectionOptions.TcpClient.Client.RemoteEndPoint.ToString();
_logger = LoggingSource.Instance.GetLogger <SubscriptionConnection>(connectionOptions.DocumentDatabase.Name);
CancellationTokenSource =
CancellationTokenSource.CreateLinkedTokenSource(TcpConnection.DocumentDatabase.DatabaseShutdown);
_waitForMoreDocuments = new AsyncManualResetEvent(CancellationTokenSource.Token);
Stats = new SubscriptionConnectionStats();
_copiedBuffer = bufferToCopy.Clone(connectionOptions.ContextPool);
}
public static async Task <bool> ReadAsync(PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer)
{
if (parser.Read())
{
return(true);
}
var read = await peepingTomStream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Length).ConfigureAwait(false);
if (read == 0)
{
if (state.CurrentTokenType != JsonParserToken.EndObject)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
return(false);
}
parser.SetBuffer(buffer, 0, read);
return(parser.Read());
}
