예제 #1
0
        public void Handle(ReplicationMessage.ReplicaSubscriptionRequest message)
        {
            _publisher.Publish(new SystemMessage.VNodeConnectionEstablished(message.ReplicaEndPoint,
                                                                            message.Connection.ConnectionId));

            if (!CanAcceptSubscription(message))
            {
                message.Envelope.ReplyWith(
                    new ReplicationMessage.ReplicaSubscriptionRetry(_instanceId, message.SubscriptionId));
                return;
            }

            var subscription = new ReplicaSubscription(_tcpSendPublisher,
                                                       message.Connection,
                                                       message.SubscriptionId,
                                                       message.ReplicaEndPoint,
                                                       message.IsPromotable);

            foreach (var subscr in _subscriptions.Values)
            {
                if (subscr != subscription && subscr.ConnectionId == subscription.ConnectionId)
                {
                    subscr.ShouldDispose = true;
                }
            }

            if (SubscribeReplica(subscription, message.LastEpochs, message.CorrelationId, message.LogPosition,
                                 message.ChunkId))
            {
                _newSubscriptions = true;
                if (!_subscriptions.TryAdd(subscription.SubscriptionId, subscription))
                {
                    ReplicaSubscription existingSubscr;
                    _subscriptions.TryGetValue(subscription.SubscriptionId, out existingSubscr);
                    Log.Error(
                        "There is already a subscription with SubscriptionID {subscriptionId:B}: {existingSubscription}.",
                        subscription.SubscriptionId, existingSubscr);
                    Log.Error("Subscription we tried to add: {existingSubscription}.", existingSubscr);
                    subscription.SendBadRequestAndClose(message.CorrelationId, string.Format(
                                                            "There is already a subscription with SubscriptionID {0:B}: {1}.\nSubscription we tried to add: {2}",
                                                            subscription.SubscriptionId, existingSubscr, subscription));
                    subscription.Dispose();
                }
                else
                {
                    var replicationCheckpoint = _db.Config.ReplicationCheckpoint.Read();
                    subscription.SendMessage(new ReplicationTrackingMessage.ReplicatedTo(replicationCheckpoint));
                }
            }
        }
예제 #2
0
        private bool SubscribeReplica(ReplicaSubscription replica, Epoch[] lastEpochs, Guid correlationId, long logPosition, Guid chunkId)
        {
            try
            {
                var epochs = lastEpochs ?? new Epoch[0];
                Log.Info("SUBSCRIBE REQUEST from [{replicaEndPoint},C:{connectionId:B},S:{subscriptionId:B},{logPosition}(0x{logPosition:X}),{epochs}]...",
                         replica.ReplicaEndPoint, replica.ConnectionId, replica.SubscriptionId, logPosition, logPosition,
                         string.Join(", ", epochs.Select(x => EpochRecordExtensions.AsString((Epoch)x))));

                var epochCorrectedLogPos = GetValidLogPosition(logPosition, epochs, replica.ReplicaEndPoint, replica.SubscriptionId);
                var subscriptionPos      = SetSubscriptionPosition(replica, epochCorrectedLogPos, chunkId,
                                                                   replicationStart: true, verbose: true, trial: 0);
                Interlocked.Exchange(ref replica.AckedLogPosition, subscriptionPos);
                return(true);
            }
            catch (Exception exc)
            {
                Log.ErrorException(exc, "Exception while subscribing replica. Connection will be dropped.");
                replica.SendBadRequestAndClose(correlationId, string.Format("Exception while subscribing replica. Connection will be dropped. Error: {0}", exc.Message));
                return(false);
            }
        }
예제 #3
0
        public void Handle(ReplicationMessage.ReplicaSubscriptionRequest message)
        {
            _publisher.Publish(new SystemMessage.VNodeConnectionEstablished(message.ReplicaEndPoint, message.Connection.ConnectionId));

            if (_state != VNodeState.Master || message.MasterId != _instanceId)
            {
                message.Envelope.ReplyWith(new ReplicationMessage.ReplicaSubscriptionRetry(_instanceId, message.SubscriptionId));
                return;
            }

            var subscription = new ReplicaSubscription(_tcpSendPublisher,
                                                       message.Connection,
                                                       message.SubscriptionId,
                                                       message.ReplicaEndPoint,
                                                       message.IsPromotable);

            foreach (var subscr in _subscriptions.Values)
            {
                if (subscr != subscription && subscr.ConnectionId == subscription.ConnectionId)
                {
                    subscr.ShouldDispose = true;
                }
            }

            if (SubscribeReplica(subscription, message.LastEpochs, message.CorrelationId, message.LogPosition, message.ChunkId))
            {
                _newSubscriptions = true;
                if (!_subscriptions.TryAdd(subscription.SubscriptionId, subscription))
                {
                    ReplicaSubscription existingSubscr;
                    _subscriptions.TryGetValue(subscription.SubscriptionId, out existingSubscr);
                    var msg = string.Format("There is already subscription with SubscriptionID {0:B}: {1}.\nSubscription we tried to add: {2}",
                                            subscription.SubscriptionId, existingSubscr, subscription);
                    Log.Error(msg);
                    subscription.SendBadRequestAndClose(message.CorrelationId, msg);
                    subscription.Dispose();
                }
            }
        }
예제 #4
0
        private bool TrySendLogBulk(ReplicaSubscription subscription, long masterCheckpoint)
        {
            /*
             * if (subscription == null) throw new Exception("subscription == null");
             * if (subscription.BulkReader == null) throw new Exception("subscription.BulkReader == null");
             * if (subscription.BulkReader.Chunk == null) throw new Exception("subscription.BulkReader.Chunk == null");
             * if (subscription.DataBuffer == null) throw new Exception("subscription.DataBuffer == null");
             */

            var bulkReader  = subscription.BulkReader;
            var chunkHeader = bulkReader.Chunk.ChunkHeader;

            BulkReadResult bulkResult;

            if (subscription.RawSend)
            {
                bulkResult = bulkReader.ReadNextRawBytes(subscription.DataBuffer.Length, subscription.DataBuffer);
            }
            else
            {
                var bytesToRead = (int)Math.Min(subscription.DataBuffer.Length, masterCheckpoint - subscription.LogPosition);
                bulkResult = bulkReader.ReadNextDataBytes(bytesToRead, subscription.DataBuffer);
            }

            bool dataFound = false;

            // for logical send we can get 0 at the end multiple time, but we need to get EOF exactly once
            if (bulkResult.BytesRead > 0 || (bulkResult.IsEOF && !subscription.RawSend && !subscription.EOFSent))
            {
                var data = new byte[bulkResult.BytesRead];
                Buffer.BlockCopy(subscription.DataBuffer, 0, data, 0, bulkResult.BytesRead);

                dataFound            = true;
                subscription.EOFSent = bulkResult.IsEOF;

                if (subscription.RawSend)
                {
                    var msg = new ReplicationMessage.RawChunkBulk(
                        _instanceId, subscription.SubscriptionId, chunkHeader.ChunkStartNumber, chunkHeader.ChunkEndNumber,
                        bulkResult.OldPosition, data, bulkResult.IsEOF);
                    subscription.SendMessage(msg);
                }
                else
                {
                    if (chunkHeader.GetLocalLogPosition(subscription.LogPosition) != bulkResult.OldPosition)
                    {
                        throw new Exception(string.Format("Replication invariant failure. SubscriptionPosition {0}, bulkResult.OldPosition {1}",
                                                          subscription.LogPosition, bulkResult.OldPosition));
                    }
                    var msg = new ReplicationMessage.DataChunkBulk(
                        _instanceId, subscription.SubscriptionId, chunkHeader.ChunkStartNumber, chunkHeader.ChunkEndNumber,
                        subscription.LogPosition, data, bulkResult.IsEOF);
                    subscription.LogPosition += bulkResult.BytesRead;
                    subscription.SendMessage(msg);
                }
            }

            if (bulkResult.IsEOF)
            {
                var newLogPosition = chunkHeader.ChunkEndPosition;
                if (newLogPosition < masterCheckpoint)
                {
                    dataFound = true;
                    SetSubscriptionPosition(subscription, newLogPosition, Guid.Empty, replicationStart: false, verbose: true, trial: 0);
                }
            }
            return(dataFound);
        }
예제 #5
0
        private long SetSubscriptionPosition(ReplicaSubscription sub,
                                             long logPosition,
                                             Guid chunkId,
                                             bool replicationStart,
                                             bool verbose,
                                             int trial)
        {
            if (trial >= 10)
            {
                throw new Exception("Too many retrials to acquire reader for subscriber.");
            }

            try
            {
                var chunk = _db.Manager.GetChunkFor(logPosition);
                Debug.Assert(chunk != null, string.Format("Chunk for LogPosition {0} (0x{0:X}) is null in MasterReplicationService! Replica: [{1},C:{2},S:{3}]",
                                                          logPosition, sub.ReplicaEndPoint, sub.ConnectionId, sub.SubscriptionId));
                var bulkReader = chunk.AcquireReader();
                if (chunk.ChunkHeader.IsScavenged && (chunkId == Guid.Empty || chunkId != chunk.ChunkHeader.ChunkId))
                {
                    var chunkStartPos = chunk.ChunkHeader.ChunkStartPosition;
                    if (verbose)
                    {
                        Log.Info("Subscribed replica [{replicaEndPoint}, S:{subscriptionId}] for raw send at {chunkStartPosition} (0x{chunkStartPosition:X}) (requested {logPosition} (0x{logPosition:X})).",
                                 sub.ReplicaEndPoint, sub.SubscriptionId, chunkStartPos, chunkStartPos, logPosition, logPosition);
                        if (chunkStartPos != logPosition)
                        {
                            Log.Info("Forcing replica [{replicaEndPoint}, S:{subscriptionId}] to recreate chunk from position {chunkStartPosition} (0x{chunkStartPosition:X})...",
                                     sub.ReplicaEndPoint, sub.SubscriptionId, chunkStartPos, chunkStartPos);
                        }
                    }

                    sub.LogPosition = chunkStartPos;
                    sub.RawSend     = true;
                    bulkReader.SetRawPosition(ChunkHeader.Size);
                    if (replicationStart)
                    {
                        sub.SendMessage(new ReplicationMessage.ReplicaSubscribed(_instanceId, sub.SubscriptionId, sub.LogPosition));
                    }
                    sub.SendMessage(new ReplicationMessage.CreateChunk(_instanceId,
                                                                       sub.SubscriptionId,
                                                                       chunk.ChunkHeader,
                                                                       chunk.FileSize,
                                                                       isCompletedChunk: true));
                }
                else
                {
                    if (verbose)
                    {
                        Log.Info("Subscribed replica [{replicaEndPoint},S:{subscriptionId}] for data send at {logPosition} (0x{logPosition:X}).", sub.ReplicaEndPoint, sub.SubscriptionId, logPosition, logPosition);
                    }

                    sub.LogPosition = logPosition;
                    sub.RawSend     = false;
                    bulkReader.SetDataPosition(chunk.ChunkHeader.GetLocalLogPosition(logPosition));
                    if (replicationStart)
                    {
                        sub.SendMessage(new ReplicationMessage.ReplicaSubscribed(_instanceId, sub.SubscriptionId, sub.LogPosition));
                    }
                }

                sub.EOFSent = false;
                var oldBulkReader = Interlocked.Exchange(ref sub.BulkReader, bulkReader);
                if (oldBulkReader != null)
                {
                    oldBulkReader.Release();
                }
                return(sub.LogPosition);
            }
            catch (FileBeingDeletedException)
            {
                return(SetSubscriptionPosition(sub, logPosition, chunkId, replicationStart, verbose, trial + 1));
            }
        }