//TODO: move it into easily testable class
public static bool ShouldBeDelivered(ReplicaId origin, VClock local, VClock remote)
{
var lver = local.Value;
foreach ((var rkey, var rtime) in remote.Value)
{
if (lver.TryGetValue(rkey, out var ltime))
{
if (rkey == origin)
{
if (rtime != ltime + 1)
{
return(false);
}
}
else
{
if (rtime > ltime)
{
return(false);
}
}
}
else if (!(rkey == origin && rtime == 1))
{
return(false);
}
}
return(true);
}
public JsonRow(string streamId, ReplicaId replicaId, VClock timestamp, string jsonPayload)
{
StreamId = streamId;
ReplicaId = replicaId;
Timestamp = timestamp;
JsonPayload = jsonPayload;
}
public ReplicatorState <T> WithLocalVersion(VClock localVersion) =>
new ReplicatorState <T>(
localVersion: localVersion,
stableVersion: StableVersion,
remoteVersions: RemoteVersions,
pendingDeliveries: PendingDeliveries,
pendingAcks: PendingAcks);
public PayloadRow(string streamId, ReplicaId replicaId, VClock timestamp, int serializerId, byte[] payload)
{
StreamId = streamId;
ReplicaId = replicaId;
Timestamp = timestamp;
SerializerId = serializerId;
Payload = payload;
}
public void VectorTime_should_be_mergeable()
{
var t1 = new VClock(new KeyValuePair <int, long>(A, 1), new KeyValuePair <int, long>(B, 2), new KeyValuePair <int, long>(C, 2));
var t2 = new VClock(new KeyValuePair <int, long>(A, 4), new KeyValuePair <int, long>(C, 1));
t1.Merge(t2).Should().Be(new VClock(new KeyValuePair <int, long>(A, 4), new KeyValuePair <int, long>(B, 2), new KeyValuePair <int, long>(C, 2)));
t2.Merge(t1).Should().Be(new VClock(new KeyValuePair <int, long>(A, 4), new KeyValuePair <int, long>(B, 2), new KeyValuePair <int, long>(C, 2)));
}
private void Assert(VClock t1, VClock t2, bool eq, bool conc, bool lt, bool lteq, bool gt, bool gteq)
{
(t1 == t2).Should().Be(eq, "{0} should be equal to {1}", t1, t2);
(t1.IsConcurrent(t2)).Should().Be(conc, "{0} should be concurrent to {1}", t1, t2);
(t1 < t2).Should().Be(lt, "{0} should be less than {1}", t1, t2);
(t1 <= t2).Should().Be(lteq, "{0} should be less or equal to {1}", t1, t2);
(t1 > t2).Should().Be(gt, "{0} should be greater than {1}", t1, t2);
(t1 >= t2).Should().Be(gteq, "{0} should be greater or equal to {1}", t1, t2);
}
public ReplicatorState(
VClock localVersion,
VClock stableVersion,
ImmutableDictionary <ReplicaId, VClock> remoteVersions,
ImmutableHashSet <Deliver <T> > pendingDeliveries,
ImmutableHashSet <PendingAck <T> > pendingAcks)
{
LocalVersion = localVersion;
StableVersion = stableVersion;
RemoteVersions = remoteVersions;
PendingDeliveries = pendingDeliveries;
PendingAcks = pendingAcks;
}
/// <summary>
/// Check and possibly deliver a message if it should be delivered.
/// </summary>
/// <returns></returns>
private bool TryToCasuallyDeliver(Deliver <T> delivery)
{
if (ShouldBeDelivered(delivery.Origin, localVersion, delivery.Versioned.Version))
{
foreach (var subscriber in subscribers)
{
subscriber.Tell(delivery.Versioned);
}
pendingDelivery = pendingDelivery.Remove(delivery);
localVersion = localVersion.Increment(delivery.Origin);
return(true);
}
return(false);
}
public void VectorTime_should_conform_to_partial_ordering()
{
var t1 = new VClock(new KeyValuePair <int, long>(A, 1), new KeyValuePair <int, long>(B, 2));
var t2 = new VClock(new KeyValuePair <int, long>(A, 1), new KeyValuePair <int, long>(B, 1));
var t3 = new VClock(new KeyValuePair <int, long>(A, 2), new KeyValuePair <int, long>(B, 1));
var t4 = new VClock(new KeyValuePair <int, long>(A, 1), new KeyValuePair <int, long>(B, 2), new KeyValuePair <int, long>(C, 2));
var t5 = new VClock(new KeyValuePair <int, long>(A, 1), new KeyValuePair <int, long>(C, 2));
var t6 = new VClock(new KeyValuePair <int, long>(A, 1), new KeyValuePair <int, long>(C, 0));
Assert(t1, t1, eq: true, conc: false, lt: false, lteq: true, gt: false, gteq: true);
Assert(t1, t2, eq: false, conc: false, lt: false, lteq: false, gt: true, gteq: true);
Assert(t2, t1, eq: false, conc: false, lt: true, lteq: true, gt: false, gteq: false);
Assert(t1, t3, eq: false, conc: true, lt: false, lteq: false, gt: false, gteq: false);
Assert(t3, t1, eq: false, conc: true, lt: false, lteq: false, gt: false, gteq: false);
Assert(t1, t4, eq: false, conc: false, lt: true, lteq: true, gt: false, gteq: false);
Assert(t4, t1, eq: false, conc: false, lt: false, lteq: false, gt: true, gteq: true);
Assert(t1, t5, eq: false, conc: true, lt: false, lteq: false, gt: false, gteq: false);
Assert(t5, t1, eq: false, conc: true, lt: false, lteq: false, gt: false, gteq: false);
Assert(t1, t6, eq: false, conc: false, lt: false, lteq: false, gt: true, gteq: true);
Assert(t6, t1, eq: false, conc: false, lt: true, lteq: true, gt: false, gteq: false);
}
public ReplicatorActor(ReplicaId myself, ReplicatorSettings settings)
{
this.myself = myself;
this.settings = settings;
replicatorRelativePath = Self.Path.ToStringWithoutAddress();
Receive <ClusterEvent.MemberUp>(up =>
{
if (HasRole(up.Member) && up.Member.Address != cluster.SelfAddress)
{
// send invitation to a new member
var path = up.Member.Address.ToString() + replicatorRelativePath;
Context.ActorSelection(path).Tell(new Invitation(myself, Self));
}
});
Receive <ClusterEvent.IMemberEvent>(_ => { /* ignore */ });
Receive <Broadcast <T> >(bcast =>
{
var version = localVersion.Increment(myself);
var versioned = new Versioned <T>(version, bcast.Message);
var send = new Send <T>(myself, myself, versioned);
if (log.IsInfoEnabled)
{
log.Info("Sending {0} to: {1}", send, string.Join(", ", members));
}
foreach (var member in members)
{
member.Value.Forward(send);
}
var pendingAck = new PendingAck <T>(myself, versioned, DateTime.UtcNow, members.Keys.ToImmutableHashSet());
this.pendingAcks = pendingAcks.Add(pendingAck);
this.localVersion = version;
});
Receive <Send <T> >(send =>
{
if (AlreadySeen(send.Versioned.Version))
{
log.Info("Received duplicate message {0}", send);
}
else
{
var receivers = members.Remove(send.Origin);
var forward = send.WithLastSeenBy(myself);
if (log.IsInfoEnabled)
{
log.Info("Broadcasting message {0} to: {1}", forward, string.Join(", ", receivers.Values));
}
foreach (var member in receivers.Values)
{
member.Forward(forward);
}
if (members.TryGetValue(send.LastSeenBy, out var sender))
{
sender.Tell(new SendAck(myself, send.Versioned.Version), ActorRefs.NoSender);
}
var deliver = new Deliver <T>(send.Origin, send.Versioned);
Self.Forward(deliver);
this.pendingDelivery = pendingDelivery.Add(deliver);
this.pendingAcks = pendingAcks.Add(new PendingAck <T>(myself, send.Versioned, DateTime.UtcNow, receivers.Keys.ToImmutableHashSet()));
}
});
Receive <SendAck>(ack =>
{
log.Info("Received ACK from {0} (version: {1})", Sender, ack.Version);
var pendingAck = this.pendingAcks.First(x => x.Versioned.Version == ack.Version);
var membersLeft = pendingAck.Members.Remove(ack.ReplicaId);
this.pendingAcks = pendingAcks.Remove(pendingAck);
if (!membersLeft.IsEmpty)
{
this.pendingAcks = pendingAcks.Add(pendingAck.WithMembers(membersLeft));
}
});
Receive <Deliver <T> >(deliver =>
{
TryToCasuallyDeliver(deliver);
remoteVersions = remoteVersions.SetItem(deliver.Origin, deliver.Versioned.Version);
latestStableVersion = UpdateStableVersion(remoteVersions);
});
Receive <Resend>(_ =>
{
var now = DateTime.UtcNow;
var builder = pendingAcks.ToBuilder();
foreach (var ack in pendingAcks)
{
if (now - ack.Timestamp > settings.RetryTimeout)
{
builder.Remove(ack);
var send = new Send <T>(myself, myself, ack.Versioned);
foreach (var replicaId in ack.Members)
{
if (members.TryGetValue(replicaId, out var member))
{
member.Tell(send, ActorRefs.NoSender);
}
}
builder.Add(ack.WithTimestamp(now));
}
}
pendingAcks = builder.ToImmutable();
});
Receive <Invitation>(invitation =>
{
log.Info("Received invitation from {0} (ID: {1})", invitation.ReplicatorRef, invitation.ReplicaId);
members = members.Add(invitation.ReplicaId, invitation.ReplicatorRef);
Context.Watch(invitation.ReplicatorRef);
});
Receive <StableReq>(sync =>
{
var reply = sync.Versions.Where(ver => latestStableVersion >= ver).ToArray();
Sender.Tell(new StableRep(reply));
});
Receive <Subscribe <T> >(subscribe =>
{
subscribers = subscribers.Add(subscribe.Ref);
if (subscribe.Ack != null)
{
subscribe.Ref.Tell(subscribe.Ack);
}
});
Receive <Unsubscribe>(unsubscribe =>
{
subscribers = subscribers.Remove(unsubscribe.Ref);
if (unsubscribe.Ack != null)
{
unsubscribe.Ref.Tell(unsubscribe.Ack);
}
});
Receive <Terminated>(terminated =>
{
subscribers = subscribers.Remove(terminated.ActorRef);
var replicaId = members.FirstOrDefault(kv => Equals(kv.Value, terminated.ActorRef));
members = members.Remove(replicaId.Key);
});
resendTask = Context.System.Scheduler
.ScheduleTellOnceCancelable(settings.ResendInterval, Self, Resend.Instance, ActorRefs.NoSender);
}
private bool AlreadySeen(VClock version) => localVersion >= version || pendingDelivery.Any(x => x.Versioned.Version == version);
public Versioned(VClock version, T message)
{
Version = version;
Message = message;
}
public SendAck(ReplicaId replicaId, VClock version)
{
ReplicaId = replicaId;
Version = version;
}
