public Delta(Bucket[] buckets)
{
Buckets = buckets ?? new Bucket[0];
}
public DistributedPubSubMediator(DistributedPubSubSettings settings)
{
if (settings.RoutingLogic is ConsistentHashingRoutingLogic)
throw new ArgumentException("Consistent hashing routing logic cannot be used by the pub-sub mediator");
_settings = settings;
if (!string.IsNullOrEmpty(_settings.Role) && !_cluster.SelfRoles.Contains(_settings.Role))
throw new ArgumentException(string.Format("The cluster member [{0}] doesn't have the role [{1}]", _cluster.SelfAddress, _settings.Role));
//Start periodic gossip to random nodes in cluster
_gossipCancelable = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.GossipInterval, _settings.GossipInterval, Self, GossipTick.Instance, Self);
_pruneInterval = new TimeSpan(_settings.RemovedTimeToLive.Ticks / 2);
_pruneCancelable = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_pruneInterval, _pruneInterval, Self, Prune.Instance, Self);
Receive<Send>(send =>
{
var routees = new List<Routee>();
Bucket bucket;
if (_registry.TryGetValue(_cluster.SelfAddress, out bucket))
{
ValueHolder valueHolder;
if (bucket.Content.TryGetValue(send.Path, out valueHolder) && send.LocalAffinity)
{
var routee = valueHolder.Routee;
if (routee != null) routees.Add(routee);
}
else
{
foreach (var entry in _registry)
{
if (entry.Value.Content.TryGetValue(send.Path, out valueHolder))
{
var routee = valueHolder.Routee;
if (routee != null) routees.Add(routee);
}
}
}
}
if (routees.Count != 0)
{
new Router(_settings.RoutingLogic, routees.ToArray()).Route(Utils.WrapIfNeeded(send.Message), Sender);
}
});
Receive<SendToAll>(sendToAll =>
{
PublishMessage(sendToAll.Path, sendToAll.Message, sendToAll.ExcludeSelf);
});
Receive<Publish>(publish =>
{
var topic = Uri.EscapeDataString(publish.Topic);
var path = Self.Path / topic;
if (publish.SendOneMessageToEachGroup)
PublishToEachGroup(path.ToStringWithoutAddress(), publish.Message);
else
PublishMessage(path.ToStringWithoutAddress(), publish.Message);
});
Receive<Put>(put =>
{
if (!string.IsNullOrEmpty(put.Ref.Path.Address.Host))
Log.Warning("Registered actor must be local: [{0}]", put.Ref);
else
{
PutToRegistry(put.Ref.Path.ToStringWithoutAddress(), put.Ref);
Context.Watch(put.Ref);
}
});
Receive<Remove>(remove =>
{
Bucket bucket;
if (_registry.TryGetValue(_cluster.SelfAddress, out bucket))
{
ValueHolder valueHolder;
if (bucket.Content.TryGetValue(remove.Path, out valueHolder) && valueHolder.Ref != null)
{
Context.Unwatch(valueHolder.Ref);
PutToRegistry(remove.Path, null);
}
}
});
Receive<Subscribe>(subscribe =>
{
// each topic is managed by a child actor with the same name as the topic
var topic = Uri.EscapeDataString(subscribe.Topic);
var child = Context.Child(topic);
if (!ActorRefs.Nobody.Equals(child))
{
child.Forward(subscribe);
}
else
{
var t = Context.ActorOf(Actor.Props.Create(() =>
new Topic(_settings.RemovedTimeToLive, _settings.RoutingLogic)), topic);
t.Forward(subscribe);
HandleRegisterTopic(t);
}
});
Receive<RegisterTopic>(register =>
{
HandleRegisterTopic(register.TopicRef);
});
Receive<GetTopics>(getTopics =>
{
Sender.Tell(new CurrentTopics(GetCurrentTopics().ToArray()));
});
Receive<Subscribed>(subscribed =>
{
subscribed.Subscriber.Tell(subscribed.Ack);
});
Receive<Unsubscribe>(unsubscribe =>
{
var topic = Uri.EscapeDataString(unsubscribe.Topic);
var child = Context.Child(topic);
if (!ActorRefs.Nobody.Equals(child))
child.Forward(unsubscribe);
});
Receive<Unsubscribed>(unsubscribed =>
{
unsubscribed.Subscriber.Tell(unsubscribed.Ack);
});
Receive<Status>(status =>
{
// gossip chat starts with a Status message, containing the bucket versions of the other node
var delta = CollectDelta(status.Versions).ToArray();
if (delta.Length != 0)
Sender.Tell(new Delta(delta));
if (OtherHasNewerVersions(status.Versions))
Sender.Tell(new Status(OwnVersions));
});
Receive<Delta>(delta =>
{
// reply from Status message in the gossip chat
// the Delta contains potential updates (newer versions) from the other node
// only accept deltas/buckets from known nodes, otherwise there is a risk of
// adding back entries when nodes are removed
if (_nodes.Contains(Sender.Path.Address))
{
foreach (var bucket in delta.Buckets)
{
if (_nodes.Contains(bucket.Owner))
{
Bucket myBucket;
if (!_registry.TryGetValue(bucket.Owner, out myBucket))
myBucket = new Bucket(bucket.Owner);
if (bucket.Version > myBucket.Version)
{
_registry.Add(bucket.Owner, new Bucket(myBucket.Owner, bucket.Version, myBucket.Content.AddRange(bucket.Content)));
}
}
}
}
});
Receive<GossipTick>(_ => HandleGossip());
Receive<Prune>(_ => HandlePrune());
Receive<Terminated>(terminated =>
{
var key = terminated.ActorRef.Path.ToStringWithoutAddress();
Bucket bucket;
if (_registry.TryGetValue(_cluster.SelfAddress, out bucket))
{
ValueHolder holder;
if (bucket.Content.TryGetValue(key, out holder) && terminated.ActorRef.Equals(holder.Ref))
{
PutToRegistry(key, null); // remove
}
}
});
Receive<ClusterEvent.CurrentClusterState>(state =>
{
var nodes = state.Members
.Where(m => m.Status != MemberStatus.Joining && IsMatchingRole(m))
.Select(m => m.Address);
_nodes = new HashSet<Address>(nodes);
});
Receive<ClusterEvent.MemberUp>(up =>
{
if (IsMatchingRole(up.Member)) _nodes.Add(up.Member.Address);
});
Receive<ClusterEvent.MemberRemoved>(removed =>
{
var member = removed.Member;
if (member.Address == _cluster.SelfAddress)
{
Context.Stop(Self);
}
else if (IsMatchingRole(member))
{
_nodes.Remove(member.Address);
_registry.Remove(member.Address);
}
});
Receive<ClusterEvent.IMemberEvent>(_ => { /* ignore */ });
Receive<Count>(_ =>
{
var count = _registry.Sum(entry => entry.Value.Content.Count(kv => kv.Value.Ref != null));
Sender.Tell(count);
});
}
private IEnumerable<Bucket> CollectDelta(IDictionary<Address, long> versions)
{
// missing entries are represented by version 0
var filledOtherVersions = new Dictionary<Address, long>(versions);
foreach (var entry in OwnVersions)
if (filledOtherVersions.ContainsKey(entry.Key))
filledOtherVersions[entry.Key] = 0L;
else
filledOtherVersions.Add(entry.Key, 0L);
var count = 0;
foreach (var entry in filledOtherVersions)
{
var owner = entry.Key;
var v = entry.Value;
Bucket bucket;
if (!_registry.TryGetValue(owner, out bucket))
bucket = new Bucket(owner);
if (bucket.Version > v && count < _settings.MaxDeltaElements)
{
var deltaContent = bucket.Content
.Where(kv => kv.Value.Version > v)
.Aggregate(ImmutableDictionary<string, ValueHolder>.Empty,
(current, kv) => current.SetItem(kv.Key, kv.Value));
count += deltaContent.Count;
if (count <= _settings.MaxDeltaElements)
yield return new Bucket(bucket.Owner, bucket.Version, deltaContent);
else
{
// exceeded the maxDeltaElements, pick the elements with lowest versions
var sortedContent = deltaContent.OrderBy(x => x.Value.Version).ToArray();
var chunk = sortedContent.Take(_settings.MaxDeltaElements - (count - sortedContent.Length)).ToList();
var content = chunk.Aggregate(ImmutableDictionary<string, ValueHolder>.Empty,
(current, kv) => current.SetItem(kv.Key, kv.Value));
yield return new Bucket(bucket.Owner, chunk.Last().Value.Version, content);
}
}
}
}
public DistributedPubSubMediator(DistributedPubSubSettings settings)
{
if (settings.RoutingLogic is ConsistentHashingRoutingLogic)
throw new ArgumentException("Consistent hashing routing logic cannot be used by the pub-sub mediator");
_settings = settings;
if (!string.IsNullOrEmpty(_settings.Role) && !_cluster.SelfRoles.Contains(_settings.Role))
throw new ArgumentException(string.Format("The cluster member [{0}] doesn't have the role [{1}]", _cluster.SelfAddress, _settings.Role));
//Start periodic gossip to random nodes in cluster
_gossipCancelable = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.GossipInterval, _settings.GossipInterval, Self, GossipTick.Instance, Self);
_pruneInterval = new TimeSpan(_settings.RemovedTimeToLive.Ticks / 2);
_pruneCancelable = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_pruneInterval, _pruneInterval, Self, Prune.Instance, Self);
_buffer = new PerGroupingBuffer();
Receive<Send>(send =>
{
var routees = new List<Routee>();
Bucket bucket;
ValueHolder valueHolder;
if (_registry.TryGetValue(_cluster.SelfAddress, out bucket) && bucket.Content.TryGetValue(send.Path, out valueHolder) && send.LocalAffinity)
{
var routee = valueHolder.Routee;
if (routee != null) routees.Add(routee);
}
else
{
foreach (var entry in _registry)
{
if (entry.Value.Content.TryGetValue(send.Path, out valueHolder))
{
var routee = valueHolder.Routee;
if (routee != null) routees.Add(routee);
}
}
}
if (routees.Count != 0)
{
new Router(_settings.RoutingLogic, routees.ToArray()).Route(
Internal.Utils.WrapIfNeeded(send.Message), Sender);
}
else
{
SendToDeadLetters(send.Message);
}
});
Receive<SendToAll>(sendToAll =>
{
PublishMessage(sendToAll.Path, sendToAll.Message, sendToAll.ExcludeSelf);
});
Receive<Publish>(publish =>
{
string path = Internal.Utils.MakeKey(Self.Path / Internal.Utils.EncodeName(publish.Topic));
if (publish.SendOneMessageToEachGroup)
PublishToEachGroup(path, publish.Message);
else
PublishMessage(path, publish.Message);
});
Receive<Put>(put =>
{
if (put.Ref.Path.Address.HasGlobalScope)
{
Log.Warning("Registered actor must be local: [{0}]", put.Ref);
}
else
{
PutToRegistry(Internal.Utils.MakeKey(put.Ref), put.Ref);
Context.Watch(put.Ref);
}
});
Receive<Remove>(remove =>
{
Bucket bucket;
if (_registry.TryGetValue(_cluster.SelfAddress, out bucket))
{
ValueHolder valueHolder;
if (bucket.Content.TryGetValue(remove.Path, out valueHolder) && valueHolder.Ref != null)
{
Context.Unwatch(valueHolder.Ref);
PutToRegistry(remove.Path, null);
}
}
});
Receive<Subscribe>(subscribe =>
{
// each topic is managed by a child actor with the same name as the topic
var encodedTopic = Internal.Utils.EncodeName(subscribe.Topic);
_buffer.BufferOr(Internal.Utils.MakeKey(Self.Path / encodedTopic), subscribe, Sender, () =>
{
var child = Context.Child(encodedTopic);
if (!child.IsNobody())
{
child.Forward(subscribe);
}
else
{
NewTopicActor(encodedTopic).Forward(subscribe);
}
});
});
Receive<RegisterTopic>(register =>
{
HandleRegisterTopic(register.TopicRef);
});
Receive<NoMoreSubscribers>(msg =>
{
var key = Internal.Utils.MakeKey(Sender);
_buffer.InitializeGrouping(key);
Sender.Tell(TerminateRequest.Instance);
});
Receive<NewSubscriberArrived>(msg =>
{
var key = Internal.Utils.MakeKey(Sender);
_buffer.ForwardMessages(key, Sender);
});
Receive<GetTopics>(getTopics =>
{
Sender.Tell(new CurrentTopics(GetCurrentTopics().ToArray()));
});
Receive<Subscribed>(subscribed =>
{
subscribed.Subscriber.Tell(subscribed.Ack);
});
Receive<Unsubscribe>(unsubscribe =>
{
var encodedTopic = Internal.Utils.EncodeName(unsubscribe.Topic);
_buffer.BufferOr(Internal.Utils.MakeKey(Self.Path / encodedTopic), unsubscribe, Sender, () =>
{
var child = Context.Child(encodedTopic);
if (!child.IsNobody())
{
child.Forward(unsubscribe);
}
else
{
// no such topic here
}
});
});
Receive<Unsubscribed>(unsubscribed =>
{
unsubscribed.Subscriber.Tell(unsubscribed.Ack);
});
Receive<Status>(status =>
{
// only accept status from known nodes, otherwise old cluster with same address may interact
// also accept from local for testing purposes
if (_nodes.Contains(Sender.Path.Address) || Sender.Path.Address.HasLocalScope)
{
// gossip chat starts with a Status message, containing the bucket versions of the other node
var delta = CollectDelta(status.Versions).ToArray();
if (delta.Length != 0)
Sender.Tell(new Delta(delta));
if (!status.IsReplyToStatus && OtherHasNewerVersions(status.Versions))
Sender.Tell(new Status(versions: OwnVersions, isReplyToStatus: true)); // it will reply with Delta
}
});
Receive<Delta>(delta =>
{
deltaCount += 1;
// reply from Status message in the gossip chat
// the Delta contains potential updates (newer versions) from the other node
// only accept deltas/buckets from known nodes, otherwise there is a risk of
// adding back entries when nodes are removed
if (_nodes.Contains(Sender.Path.Address))
{
foreach (var bucket in delta.Buckets)
{
if (_nodes.Contains(bucket.Owner))
{
Bucket myBucket;
if (!_registry.TryGetValue(bucket.Owner, out myBucket))
myBucket = new Bucket(bucket.Owner);
if (bucket.Version > myBucket.Version)
{
_registry[bucket.Owner] = new Bucket(myBucket.Owner, bucket.Version, myBucket.Content.SetItems(bucket.Content));
}
}
}
}
});
Receive<GossipTick>(_ => HandleGossip());
Receive<Prune>(_ => HandlePrune());
Receive<Terminated>(terminated =>
{
var key = Internal.Utils.MakeKey(terminated.ActorRef);
Bucket bucket;
if (_registry.TryGetValue(_cluster.SelfAddress, out bucket))
{
ValueHolder holder;
if (bucket.Content.TryGetValue(key, out holder) && terminated.ActorRef.Equals(holder.Ref))
{
PutToRegistry(key, null); // remove
}
}
_buffer.RecreateAndForwardMessagesIfNeeded(key, () => NewTopicActor(terminated.ActorRef.Path.Name));
});
Receive<ClusterEvent.CurrentClusterState>(state =>
{
var nodes = state.Members
.Where(m => m.Status != MemberStatus.Joining && IsMatchingRole(m))
.Select(m => m.Address);
_nodes = new HashSet<Address>(nodes);
});
Receive<ClusterEvent.MemberUp>(up =>
{
if (IsMatchingRole(up.Member)) _nodes.Add(up.Member.Address);
});
Receive<ClusterEvent.MemberLeft>(left =>
{
if (IsMatchingRole(left.Member))
{
_nodes.Remove(left.Member.Address);
_registry.Remove(left.Member.Address);
}
});
Receive<ClusterEvent.MemberRemoved>(removed =>
{
var member = removed.Member;
if (member.Address == _cluster.SelfAddress)
{
Context.Stop(Self);
}
else if (IsMatchingRole(member))
{
_nodes.Remove(member.Address);
_registry.Remove(member.Address);
}
});
Receive<ClusterEvent.IMemberEvent>(_ => { /* ignore */ });
Receive<Count>(_ =>
{
var count = _registry.Sum(entry => entry.Value.Content.Count(kv => kv.Value.Ref != null));
Sender.Tell(count);
});
Receive<DeltaCount>(_ =>
{
Sender.Tell(deltaCount);
});
}