public void Handle(ReplicationMessage.ReplicaSubscriptionRequest message)
{
_publisher.Publish(new SystemMessage.VNodeConnectionEstablished(message.ReplicaEndPoint,
message.Connection.ConnectionId));
if (!CanAcceptSubscription(message))
{
message.Envelope.ReplyWith(
new ReplicationMessage.ReplicaSubscriptionRetry(_instanceId, message.SubscriptionId));
return;
}
var subscription = new ReplicaSubscription(_tcpSendPublisher,
message.Connection,
message.SubscriptionId,
message.ReplicaEndPoint,
message.IsPromotable);
foreach (var subscr in _subscriptions.Values)
{
if (subscr != subscription && subscr.ConnectionId == subscription.ConnectionId)
{
subscr.ShouldDispose = true;
}
}
if (SubscribeReplica(subscription, message.LastEpochs, message.CorrelationId, message.LogPosition,
message.ChunkId))
{
_newSubscriptions = true;
if (!_subscriptions.TryAdd(subscription.SubscriptionId, subscription))
{
ReplicaSubscription existingSubscr;
_subscriptions.TryGetValue(subscription.SubscriptionId, out existingSubscr);
Log.Error(
"There is already a subscription with SubscriptionID {subscriptionId:B}: {existingSubscription}.",
subscription.SubscriptionId, existingSubscr);
Log.Error("Subscription we tried to add: {existingSubscription}.", existingSubscr);
subscription.SendBadRequestAndClose(message.CorrelationId, string.Format(
"There is already a subscription with SubscriptionID {0:B}: {1}.\nSubscription we tried to add: {2}",
subscription.SubscriptionId, existingSubscr, subscription));
subscription.Dispose();
}
else
{
var replicationCheckpoint = _db.Config.ReplicationCheckpoint.Read();
subscription.SendMessage(new ReplicationTrackingMessage.ReplicatedTo(replicationCheckpoint));
}
}
}
private bool SubscribeReplica(ReplicaSubscription replica, Epoch[] lastEpochs, Guid correlationId, long logPosition, Guid chunkId)
{
try
{
var epochs = lastEpochs ?? new Epoch[0];
Log.Info("SUBSCRIBE REQUEST from [{replicaEndPoint},C:{connectionId:B},S:{subscriptionId:B},{logPosition}(0x{logPosition:X}),{epochs}]...",
replica.ReplicaEndPoint, replica.ConnectionId, replica.SubscriptionId, logPosition, logPosition,
string.Join(", ", epochs.Select(x => EpochRecordExtensions.AsString((Epoch)x))));
var epochCorrectedLogPos = GetValidLogPosition(logPosition, epochs, replica.ReplicaEndPoint, replica.SubscriptionId);
var subscriptionPos = SetSubscriptionPosition(replica, epochCorrectedLogPos, chunkId,
replicationStart: true, verbose: true, trial: 0);
Interlocked.Exchange(ref replica.AckedLogPosition, subscriptionPos);
return(true);
}
catch (Exception exc)
{
Log.ErrorException(exc, "Exception while subscribing replica. Connection will be dropped.");
replica.SendBadRequestAndClose(correlationId, string.Format("Exception while subscribing replica. Connection will be dropped. Error: {0}", exc.Message));
return(false);
}
}
public void Handle(ReplicationMessage.ReplicaSubscriptionRequest message)
{
_publisher.Publish(new SystemMessage.VNodeConnectionEstablished(message.ReplicaEndPoint, message.Connection.ConnectionId));
if (_state != VNodeState.Master || message.MasterId != _instanceId)
{
message.Envelope.ReplyWith(new ReplicationMessage.ReplicaSubscriptionRetry(_instanceId, message.SubscriptionId));
return;
}
var subscription = new ReplicaSubscription(_tcpSendPublisher,
message.Connection,
message.SubscriptionId,
message.ReplicaEndPoint,
message.IsPromotable);
foreach (var subscr in _subscriptions.Values)
{
if (subscr != subscription && subscr.ConnectionId == subscription.ConnectionId)
{
subscr.ShouldDispose = true;
}
}
if (SubscribeReplica(subscription, message.LastEpochs, message.CorrelationId, message.LogPosition, message.ChunkId))
{
_newSubscriptions = true;
if (!_subscriptions.TryAdd(subscription.SubscriptionId, subscription))
{
ReplicaSubscription existingSubscr;
_subscriptions.TryGetValue(subscription.SubscriptionId, out existingSubscr);
var msg = string.Format("There is already subscription with SubscriptionID {0:B}: {1}.\nSubscription we tried to add: {2}",
subscription.SubscriptionId, existingSubscr, subscription);
Log.Error(msg);
subscription.SendBadRequestAndClose(message.CorrelationId, msg);
subscription.Dispose();
}
}
}
private bool TrySendLogBulk(ReplicaSubscription subscription, long masterCheckpoint)
{
/*
* if (subscription == null) throw new Exception("subscription == null");
* if (subscription.BulkReader == null) throw new Exception("subscription.BulkReader == null");
* if (subscription.BulkReader.Chunk == null) throw new Exception("subscription.BulkReader.Chunk == null");
* if (subscription.DataBuffer == null) throw new Exception("subscription.DataBuffer == null");
*/
var bulkReader = subscription.BulkReader;
var chunkHeader = bulkReader.Chunk.ChunkHeader;
BulkReadResult bulkResult;
if (subscription.RawSend)
{
bulkResult = bulkReader.ReadNextRawBytes(subscription.DataBuffer.Length, subscription.DataBuffer);
}
else
{
var bytesToRead = (int)Math.Min(subscription.DataBuffer.Length, masterCheckpoint - subscription.LogPosition);
bulkResult = bulkReader.ReadNextDataBytes(bytesToRead, subscription.DataBuffer);
}
bool dataFound = false;
// for logical send we can get 0 at the end multiple time, but we need to get EOF exactly once
if (bulkResult.BytesRead > 0 || (bulkResult.IsEOF && !subscription.RawSend && !subscription.EOFSent))
{
var data = new byte[bulkResult.BytesRead];
Buffer.BlockCopy(subscription.DataBuffer, 0, data, 0, bulkResult.BytesRead);
dataFound = true;
subscription.EOFSent = bulkResult.IsEOF;
if (subscription.RawSend)
{
var msg = new ReplicationMessage.RawChunkBulk(
_instanceId, subscription.SubscriptionId, chunkHeader.ChunkStartNumber, chunkHeader.ChunkEndNumber,
bulkResult.OldPosition, data, bulkResult.IsEOF);
subscription.SendMessage(msg);
}
else
{
if (chunkHeader.GetLocalLogPosition(subscription.LogPosition) != bulkResult.OldPosition)
{
throw new Exception(string.Format("Replication invariant failure. SubscriptionPosition {0}, bulkResult.OldPosition {1}",
subscription.LogPosition, bulkResult.OldPosition));
}
var msg = new ReplicationMessage.DataChunkBulk(
_instanceId, subscription.SubscriptionId, chunkHeader.ChunkStartNumber, chunkHeader.ChunkEndNumber,
subscription.LogPosition, data, bulkResult.IsEOF);
subscription.LogPosition += bulkResult.BytesRead;
subscription.SendMessage(msg);
}
}
if (bulkResult.IsEOF)
{
var newLogPosition = chunkHeader.ChunkEndPosition;
if (newLogPosition < masterCheckpoint)
{
dataFound = true;
SetSubscriptionPosition(subscription, newLogPosition, Guid.Empty, replicationStart: false, verbose: true, trial: 0);
}
}
return(dataFound);
}
private long SetSubscriptionPosition(ReplicaSubscription sub,
long logPosition,
Guid chunkId,
bool replicationStart,
bool verbose,
int trial)
{
if (trial >= 10)
{
throw new Exception("Too many retrials to acquire reader for subscriber.");
}
try
{
var chunk = _db.Manager.GetChunkFor(logPosition);
Debug.Assert(chunk != null, string.Format("Chunk for LogPosition {0} (0x{0:X}) is null in MasterReplicationService! Replica: [{1},C:{2},S:{3}]",
logPosition, sub.ReplicaEndPoint, sub.ConnectionId, sub.SubscriptionId));
var bulkReader = chunk.AcquireReader();
if (chunk.ChunkHeader.IsScavenged && (chunkId == Guid.Empty || chunkId != chunk.ChunkHeader.ChunkId))
{
var chunkStartPos = chunk.ChunkHeader.ChunkStartPosition;
if (verbose)
{
Log.Info("Subscribed replica [{replicaEndPoint}, S:{subscriptionId}] for raw send at {chunkStartPosition} (0x{chunkStartPosition:X}) (requested {logPosition} (0x{logPosition:X})).",
sub.ReplicaEndPoint, sub.SubscriptionId, chunkStartPos, chunkStartPos, logPosition, logPosition);
if (chunkStartPos != logPosition)
{
Log.Info("Forcing replica [{replicaEndPoint}, S:{subscriptionId}] to recreate chunk from position {chunkStartPosition} (0x{chunkStartPosition:X})...",
sub.ReplicaEndPoint, sub.SubscriptionId, chunkStartPos, chunkStartPos);
}
}
sub.LogPosition = chunkStartPos;
sub.RawSend = true;
bulkReader.SetRawPosition(ChunkHeader.Size);
if (replicationStart)
{
sub.SendMessage(new ReplicationMessage.ReplicaSubscribed(_instanceId, sub.SubscriptionId, sub.LogPosition));
}
sub.SendMessage(new ReplicationMessage.CreateChunk(_instanceId,
sub.SubscriptionId,
chunk.ChunkHeader,
chunk.FileSize,
isCompletedChunk: true));
}
else
{
if (verbose)
{
Log.Info("Subscribed replica [{replicaEndPoint},S:{subscriptionId}] for data send at {logPosition} (0x{logPosition:X}).", sub.ReplicaEndPoint, sub.SubscriptionId, logPosition, logPosition);
}
sub.LogPosition = logPosition;
sub.RawSend = false;
bulkReader.SetDataPosition(chunk.ChunkHeader.GetLocalLogPosition(logPosition));
if (replicationStart)
{
sub.SendMessage(new ReplicationMessage.ReplicaSubscribed(_instanceId, sub.SubscriptionId, sub.LogPosition));
}
}
sub.EOFSent = false;
var oldBulkReader = Interlocked.Exchange(ref sub.BulkReader, bulkReader);
if (oldBulkReader != null)
{
oldBulkReader.Release();
}
return(sub.LogPosition);
}
catch (FileBeingDeletedException)
{
return(SetSubscriptionPosition(sub, logPosition, chunkId, replicationStart, verbose, trial + 1));
}
}