public RoundBasedRegister(IIntercomMessageHub intercomMessageHub,
IBallotGenerator ballotGenerator,
ISynodConfigurationProvider synodConfigProvider,
LeaseConfiguration leaseConfig,
ILogger logger)
{
this.logger = logger;
this.synodConfigProvider = synodConfigProvider;
this.leaseConfig = leaseConfig;
this.intercomMessageHub = intercomMessageHub;
readBallot = ballotGenerator.Null();
writeBallot = ballotGenerator.Null();
listener = intercomMessageHub.Subscribe();
listener.Where(m => m.Equals(ConsensusMessages.LeaseRead)).Subscribe(OnReadReceived);
listener.Where(m => m.Equals(ConsensusMessages.LeaseWrite)).Subscribe(OnWriteReceived);
ackReadStream = listener.Where(m => m.Equals(ConsensusMessages.LeaseAckRead));
nackReadStream = listener.Where(m => m.Equals(ConsensusMessages.LeaseNackRead));
ackWriteStream = listener.Where(m => m.Equals(ConsensusMessages.LeaseAckWrite));
nackWriteStream = listener.Where(m => m.Equals(ConsensusMessages.LeaseNackWrite));
WaitBeforeNextLeaseIssued(leaseConfig);
}
private void OnWriteReceived(IMessage message)
{
var payload = message.GetPayload <LeaseWriteMessage>();
var ballot = new Ballot(new DateTime(payload.Ballot.Timestamp, DateTimeKind.Utc),
payload.Ballot.MessageNumber,
payload.Ballot.Identity);
IMessage response;
if (Interlocked.Exchange(ref writeBallot, writeBallot) > ballot ||
Interlocked.Exchange(ref readBallot, readBallot) > ballot)
{
LogNackWrite(ballot);
response = Message.Create(new LeaseNackWriteMessage
{
Ballot = payload.Ballot,
SenderUri = synodConfigProvider.LocalNode.Uri.ToSocketAddress()
});
}
else
{
LogAckWrite(ballot);
Interlocked.Exchange(ref writeBallot, ballot);
Interlocked.Exchange(ref lease, new Lease(payload.Lease.Identity, new DateTime(payload.Lease.ExpiresAt, DateTimeKind.Utc), payload.Lease.OwnerPayload));
response = Message.Create(new LeaseAckWriteMessage
{
Ballot = payload.Ballot,
SenderUri = synodConfigProvider.LocalNode.Uri.ToSocketAddress()
});
}
intercomMessageHub.Send(response);
}
public InstanceRoundBasedRegister(Instance instance,
IIntercomMessageHub intercomMessageHub,
IBallotGenerator ballotGenerator,
ISynodConfigurationProvider synodConfigProvider,
InstanceLeaseProviderConfiguration leaseConfig,
ILogger logger)
{
this.instance = instance;
this.logger = logger;
this.synodConfigProvider = synodConfigProvider;
this.leaseConfig = leaseConfig;
this.intercomMessageHub = intercomMessageHub;
readBallot = ballotGenerator.Null();
writeBallot = ballotGenerator.Null();
lastIntercomeMessageTimestamp = DateTime.UtcNow.Ticks;
listener = intercomMessageHub.Subscribe();
listener.Where(IsLeaseRead)
.Subscribe(OnReadReceived);
listener.Where(IsWriteLease)
.Subscribe(OnWriteReceived);
ackReadStream = listener.Where(IsLeaseAckReadMessage);
nackReadStream = listener.Where(IsLeaseNackReadMessage);
ackWriteStream = listener.Where(IsLeaseAckWriteMessage);
nackWriteStream = listener.Where(IsLeaseNackWriteMessage);
logger.Info($"{instance.Identity.GetAnyString()}-InstanceRoundBasedRegister created");
}
private IMessage CreateLeaseAckReadMessage(LeaseReadMessage payload)
{
Lease lastKnownLease = null;
Interlocked.Exchange(ref lastKnownLease, lease);
Ballot lastKnownWriteBallot = null;
Interlocked.Exchange(ref lastKnownWriteBallot, writeBallot);
return(Message.Create(new LeaseAckReadMessage
{
Ballot = payload.Ballot,
KnownWriteBallot = new Messages.Ballot
{
Identity = lastKnownWriteBallot.Identity,
Timestamp = lastKnownWriteBallot.Timestamp.Ticks,
MessageNumber = lastKnownWriteBallot.MessageNumber
},
Lease = (lastKnownLease != null)
? new Messages.Lease
{
Identity = lastKnownLease.OwnerIdentity,
ExpiresAt = lastKnownLease.ExpiresAt.Ticks,
OwnerPayload = lastKnownLease.OwnerPayload
}
: null,
SenderUri = synodConfigProvider.LocalNode.Uri.ToSocketAddress()
}));
}
private Lease AсquireOrLearnLease(Ballot ballot, DateTime now)
{
var read = register.Read(ballot);
if (read.TxOutcome == TxOutcome.Commit)
{
var lease = read.Lease;
if (LeaseIsNotSafelyExpired(lease, now))
{
LogStartSleep();
config.ClockDrift.Sleep();
LogAwake();
// TODO: Add recursion exit condition
return(AсquireOrLearnLease(ballotGenerator.New(localNode.SocketIdentity), DateTime.UtcNow));
}
if (LeaseNullOrExpired(lease, now) || IsLeaseOwner(lease))
{
LogLeaseProlonged(lease);
lease = new Lease(localNode.SocketIdentity, now + config.MaxLeaseTimeSpan, Interlocked.Exchange(ref ownerPayload, ownerPayload));
}
var write = register.Write(ballot, lease);
if (write.TxOutcome == TxOutcome.Commit)
{
return(lease);
}
}
return(null);
}
public LeaseTxResult Write(Ballot ballot, Lease lease)
{
var ackFilter = new LeaderElectionMessageFilter(ballot, m => m.GetPayload <LeaseAckWriteMessage>(), synodConfigProvider);
var nackFilter = new LeaderElectionMessageFilter(ballot, m => m.GetPayload <LeaseNackWriteMessage>(), synodConfigProvider);
var awaitableAckFilter = new AwaitableMessageStreamFilter(ackFilter.Match, m => m.GetPayload <LeaseAckWriteMessage>(), GetQuorum());
var awaitableNackFilter = new AwaitableMessageStreamFilter(nackFilter.Match, m => m.GetPayload <LeaseNackWriteMessage>(), GetQuorum());
using (ackWriteStream.Subscribe(awaitableAckFilter))
{
using (nackWriteStream.Subscribe(awaitableNackFilter))
{
intercomMessageHub.Send(CreateWriteMessage(ballot, lease));
var index = WaitHandle.WaitAny(new[] { awaitableAckFilter.Filtered, awaitableNackFilter.Filtered },
leaseConfig.NodeResponseTimeout);
if (ReadNotAcknowledged(index))
{
return(new LeaseTxResult {
TxOutcome = TxOutcome.Abort
});
}
return(new LeaseTxResult
{
TxOutcome = TxOutcome.Commit,
// NOTE: needed???
Lease = lease
});
}
}
}
private void OnReadReceived(IMessage message)
{
Interlocked.Exchange(ref lastIntercomeMessageTimestamp, DateTime.UtcNow.Ticks);
var payload = message.GetPayload <LeaseReadMessage>();
var ballot = new Ballot(new DateTime(payload.Ballot.Timestamp, DateTimeKind.Utc),
payload.Ballot.MessageNumber,
payload.Ballot.Identity);
IMessage response;
if (Interlocked.Exchange(ref writeBallot, writeBallot) >= ballot || Interlocked.Exchange(ref readBallot, readBallot) >= ballot)
{
LogNackRead(ballot);
response = Message.Create(new LeaseNackReadMessage
{
Ballot = payload.Ballot,
SenderUri = synodConfigProvider.LocalNode.Uri.ToSocketAddress()
});
}
else
{
LogAckRead(ballot);
Interlocked.Exchange(ref readBallot, ballot);
response = CreateLeaseAckReadMessage(payload);
}
intercomMessageHub.Send(response);
}
private void LogAckWrite(Ballot ballot)
{
if (writeBallot <= ballot)
{
logger.Debug($"process {synodConfig.LocalNode.Uri.AbsoluteUri} " +
"ACK_WRITE ==WB== " +
$"{writeBallot.Timestamp.ToString(format)}-" +
$"{writeBallot.MessageNumber}-" +
$"{writeBallot.Identity.GetAnyString()} " +
"<= " +
$"{ballot.Timestamp.ToString(format)}-" +
$"{ballot.MessageNumber}-" +
$"{ballot.Identity.GetAnyString()}");
}
if (readBallot <= ballot)
{
logger.Debug($"process {synodConfig.LocalNode.Uri.AbsoluteUri} " +
"ACK_WRITE ==RB== " +
$"{readBallot.Timestamp.ToString(format)}-" +
$"{readBallot.MessageNumber}-" +
$"{readBallot.Identity.GetAnyString()} " +
"<= " +
$"{ballot.Timestamp.ToString(format)}-" +
$"{ballot.MessageNumber}-" +
$"{ballot.Identity.GetAnyString()}");
}
}
private void LogNackRead(Ballot ballot)
{
if (writeBallot >= ballot)
{
logger.Debug($"process {synodConfigProvider.LocalNode.Uri.AbsoluteUri} " +
"NACK_READ ==WB== " +
$"{writeBallot.Timestamp.ToString(format)}-" +
$"{writeBallot.MessageNumber}-" +
$"{writeBallot.Identity.GetAnyString()} " +
">= " +
$"{ballot.Timestamp.ToString(format)}-" +
$"{ballot.MessageNumber}-" +
$"{ballot.Identity.GetAnyString()}");
}
if (readBallot >= ballot)
{
logger.Debug($"process {synodConfigProvider.LocalNode.Uri.AbsoluteUri} " +
"NACK_READ ==RB== " +
$"{readBallot.Timestamp.ToString(format)}-" +
$"{readBallot.MessageNumber}-" +
$"{readBallot.Identity.GetAnyString()} " +
">= " +
$"{ballot.Timestamp.ToString(format)}-" +
$"{ballot.MessageNumber}-" +
$"{ballot.Identity.GetAnyString()}");
}
}
public LeaderElectionMessageFilter(Ballot ballot,
Func <IMessage, ILeaseMessage> payload,
ISynodConfiguration synodConfig)
{
this.ballot = ballot;
this.synodConfig = synodConfig;
this.payload = payload;
}
private IMessage CreateReadMessage(Ballot ballot)
=> Message.Create(new LeaseReadMessage
{
Ballot = new Messages.Ballot
{
Identity = ballot.Identity,
Timestamp = ballot.Timestamp.Ticks,
MessageNumber = ballot.MessageNumber
}
});
private IMessage CreateWriteMessage(Ballot ballot, Lease lease)
=> Message.Create(new LeaseWriteMessage
{
Ballot = new Messages.Ballot
{
Identity = ballot.Identity,
Timestamp = ballot.Timestamp.Ticks,
MessageNumber = ballot.MessageNumber
},
Lease = new Messages.Lease
{
Identity = lease.OwnerIdentity,
ExpiresAt = lease.ExpiresAt.Ticks,
OwnerPayload = lease.OwnerPayload
}
});
public LeaseTxResult Read(Ballot ballot)
{
var ackFilter = new LeaderElectionMessageFilter(ballot,
m => m.GetPayload <LeaseAckReadMessage>(),
synodConfigProvider);
var nackFilter = new LeaderElectionMessageFilter(ballot,
m => m.GetPayload <LeaseNackReadMessage>(),
synodConfigProvider);
var awaitableAckFilter = new AwaitableMessageStreamFilter(ackFilter.Match,
m => m.GetPayload <LeaseAckReadMessage>(),
GetQuorum());
var awaitableNackFilter = new AwaitableMessageStreamFilter(nackFilter.Match,
m => m.GetPayload <LeaseNackReadMessage>(),
GetQuorum());
using (ackReadStream.Subscribe(awaitableAckFilter))
{
using (nackReadStream.Subscribe(awaitableNackFilter))
{
var message = CreateReadMessage(ballot);
intercomMessageHub.Send(message);
var index = WaitHandle.WaitAny(new[] { awaitableAckFilter.Filtered, awaitableNackFilter.Filtered },
leaseConfig.NodeResponseTimeout);
if (ReadNotAcknowledged(index))
{
return(new LeaseTxResult {
TxOutcome = TxOutcome.Abort
});
}
var lease = awaitableAckFilter
.MessageStream
.Select(m => m.GetPayload <LeaseAckReadMessage>())
.Max(p => CreateLastWrittenLease(p))
.Lease;
return(new LeaseTxResult
{
TxOutcome = TxOutcome.Commit,
Lease = lease
});
}
}
}
public LastWrittenLease(Ballot writeBallot, Lease lease)
{
this.writeBallot = writeBallot;
Lease = lease;
}
