public Delta(Bucket[] buckets) { Buckets = buckets ?? new Bucket[0]; }
public DistributedPubSubMediator(DistributedPubSubSettings settings) { if (settings.RoutingLogic is ConsistentHashingRoutingLogic) throw new ArgumentException("Consistent hashing routing logic cannot be used by the pub-sub mediator"); _settings = settings; if (!string.IsNullOrEmpty(_settings.Role) && !_cluster.SelfRoles.Contains(_settings.Role)) throw new ArgumentException(string.Format("The cluster member [{0}] doesn't have the role [{1}]", _cluster.SelfAddress, _settings.Role)); //Start periodic gossip to random nodes in cluster _gossipCancelable = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.GossipInterval, _settings.GossipInterval, Self, GossipTick.Instance, Self); _pruneInterval = new TimeSpan(_settings.RemovedTimeToLive.Ticks / 2); _pruneCancelable = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_pruneInterval, _pruneInterval, Self, Prune.Instance, Self); Receive<Send>(send => { var routees = new List<Routee>(); Bucket bucket; if (_registry.TryGetValue(_cluster.SelfAddress, out bucket)) { ValueHolder valueHolder; if (bucket.Content.TryGetValue(send.Path, out valueHolder) && send.LocalAffinity) { var routee = valueHolder.Routee; if (routee != null) routees.Add(routee); } else { foreach (var entry in _registry) { if (entry.Value.Content.TryGetValue(send.Path, out valueHolder)) { var routee = valueHolder.Routee; if (routee != null) routees.Add(routee); } } } } if (routees.Count != 0) { new Router(_settings.RoutingLogic, routees.ToArray()).Route(Utils.WrapIfNeeded(send.Message), Sender); } }); Receive<SendToAll>(sendToAll => { PublishMessage(sendToAll.Path, sendToAll.Message, sendToAll.ExcludeSelf); }); Receive<Publish>(publish => { var topic = Uri.EscapeDataString(publish.Topic); var path = Self.Path / topic; if (publish.SendOneMessageToEachGroup) PublishToEachGroup(path.ToStringWithoutAddress(), publish.Message); else PublishMessage(path.ToStringWithoutAddress(), publish.Message); }); Receive<Put>(put => { if (!string.IsNullOrEmpty(put.Ref.Path.Address.Host)) Log.Warning("Registered actor must be local: [{0}]", put.Ref); else { PutToRegistry(put.Ref.Path.ToStringWithoutAddress(), put.Ref); Context.Watch(put.Ref); } }); Receive<Remove>(remove => { Bucket bucket; if (_registry.TryGetValue(_cluster.SelfAddress, out bucket)) { ValueHolder valueHolder; if (bucket.Content.TryGetValue(remove.Path, out valueHolder) && valueHolder.Ref != null) { Context.Unwatch(valueHolder.Ref); PutToRegistry(remove.Path, null); } } }); Receive<Subscribe>(subscribe => { // each topic is managed by a child actor with the same name as the topic var topic = Uri.EscapeDataString(subscribe.Topic); var child = Context.Child(topic); if (!ActorRefs.Nobody.Equals(child)) { child.Forward(subscribe); } else { var t = Context.ActorOf(Actor.Props.Create(() => new Topic(_settings.RemovedTimeToLive, _settings.RoutingLogic)), topic); t.Forward(subscribe); HandleRegisterTopic(t); } }); Receive<RegisterTopic>(register => { HandleRegisterTopic(register.TopicRef); }); Receive<GetTopics>(getTopics => { Sender.Tell(new CurrentTopics(GetCurrentTopics().ToArray())); }); Receive<Subscribed>(subscribed => { subscribed.Subscriber.Tell(subscribed.Ack); }); Receive<Unsubscribe>(unsubscribe => { var topic = Uri.EscapeDataString(unsubscribe.Topic); var child = Context.Child(topic); if (!ActorRefs.Nobody.Equals(child)) child.Forward(unsubscribe); }); Receive<Unsubscribed>(unsubscribed => { unsubscribed.Subscriber.Tell(unsubscribed.Ack); }); Receive<Status>(status => { // gossip chat starts with a Status message, containing the bucket versions of the other node var delta = CollectDelta(status.Versions).ToArray(); if (delta.Length != 0) Sender.Tell(new Delta(delta)); if (OtherHasNewerVersions(status.Versions)) Sender.Tell(new Status(OwnVersions)); }); Receive<Delta>(delta => { // reply from Status message in the gossip chat // the Delta contains potential updates (newer versions) from the other node // only accept deltas/buckets from known nodes, otherwise there is a risk of // adding back entries when nodes are removed if (_nodes.Contains(Sender.Path.Address)) { foreach (var bucket in delta.Buckets) { if (_nodes.Contains(bucket.Owner)) { Bucket myBucket; if (!_registry.TryGetValue(bucket.Owner, out myBucket)) myBucket = new Bucket(bucket.Owner); if (bucket.Version > myBucket.Version) { _registry.Add(bucket.Owner, new Bucket(myBucket.Owner, bucket.Version, myBucket.Content.AddRange(bucket.Content))); } } } } }); Receive<GossipTick>(_ => HandleGossip()); Receive<Prune>(_ => HandlePrune()); Receive<Terminated>(terminated => { var key = terminated.ActorRef.Path.ToStringWithoutAddress(); Bucket bucket; if (_registry.TryGetValue(_cluster.SelfAddress, out bucket)) { ValueHolder holder; if (bucket.Content.TryGetValue(key, out holder) && terminated.ActorRef.Equals(holder.Ref)) { PutToRegistry(key, null); // remove } } }); Receive<ClusterEvent.CurrentClusterState>(state => { var nodes = state.Members .Where(m => m.Status != MemberStatus.Joining && IsMatchingRole(m)) .Select(m => m.Address); _nodes = new HashSet<Address>(nodes); }); Receive<ClusterEvent.MemberUp>(up => { if (IsMatchingRole(up.Member)) _nodes.Add(up.Member.Address); }); Receive<ClusterEvent.MemberRemoved>(removed => { var member = removed.Member; if (member.Address == _cluster.SelfAddress) { Context.Stop(Self); } else if (IsMatchingRole(member)) { _nodes.Remove(member.Address); _registry.Remove(member.Address); } }); Receive<ClusterEvent.IMemberEvent>(_ => { /* ignore */ }); Receive<Count>(_ => { var count = _registry.Sum(entry => entry.Value.Content.Count(kv => kv.Value.Ref != null)); Sender.Tell(count); }); }
private IEnumerable<Bucket> CollectDelta(IDictionary<Address, long> versions) { // missing entries are represented by version 0 var filledOtherVersions = new Dictionary<Address, long>(versions); foreach (var entry in OwnVersions) if (filledOtherVersions.ContainsKey(entry.Key)) filledOtherVersions[entry.Key] = 0L; else filledOtherVersions.Add(entry.Key, 0L); var count = 0; foreach (var entry in filledOtherVersions) { var owner = entry.Key; var v = entry.Value; Bucket bucket; if (!_registry.TryGetValue(owner, out bucket)) bucket = new Bucket(owner); if (bucket.Version > v && count < _settings.MaxDeltaElements) { var deltaContent = bucket.Content .Where(kv => kv.Value.Version > v) .Aggregate(ImmutableDictionary<string, ValueHolder>.Empty, (current, kv) => current.SetItem(kv.Key, kv.Value)); count += deltaContent.Count; if (count <= _settings.MaxDeltaElements) yield return new Bucket(bucket.Owner, bucket.Version, deltaContent); else { // exceeded the maxDeltaElements, pick the elements with lowest versions var sortedContent = deltaContent.OrderBy(x => x.Value.Version).ToArray(); var chunk = sortedContent.Take(_settings.MaxDeltaElements - (count - sortedContent.Length)).ToList(); var content = chunk.Aggregate(ImmutableDictionary<string, ValueHolder>.Empty, (current, kv) => current.SetItem(kv.Key, kv.Value)); yield return new Bucket(bucket.Owner, chunk.Last().Value.Version, content); } } } }
public DistributedPubSubMediator(DistributedPubSubSettings settings) { if (settings.RoutingLogic is ConsistentHashingRoutingLogic) throw new ArgumentException("Consistent hashing routing logic cannot be used by the pub-sub mediator"); _settings = settings; if (!string.IsNullOrEmpty(_settings.Role) && !_cluster.SelfRoles.Contains(_settings.Role)) throw new ArgumentException(string.Format("The cluster member [{0}] doesn't have the role [{1}]", _cluster.SelfAddress, _settings.Role)); //Start periodic gossip to random nodes in cluster _gossipCancelable = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.GossipInterval, _settings.GossipInterval, Self, GossipTick.Instance, Self); _pruneInterval = new TimeSpan(_settings.RemovedTimeToLive.Ticks / 2); _pruneCancelable = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_pruneInterval, _pruneInterval, Self, Prune.Instance, Self); _buffer = new PerGroupingBuffer(); Receive<Send>(send => { var routees = new List<Routee>(); Bucket bucket; ValueHolder valueHolder; if (_registry.TryGetValue(_cluster.SelfAddress, out bucket) && bucket.Content.TryGetValue(send.Path, out valueHolder) && send.LocalAffinity) { var routee = valueHolder.Routee; if (routee != null) routees.Add(routee); } else { foreach (var entry in _registry) { if (entry.Value.Content.TryGetValue(send.Path, out valueHolder)) { var routee = valueHolder.Routee; if (routee != null) routees.Add(routee); } } } if (routees.Count != 0) { new Router(_settings.RoutingLogic, routees.ToArray()).Route( Internal.Utils.WrapIfNeeded(send.Message), Sender); } else { SendToDeadLetters(send.Message); } }); Receive<SendToAll>(sendToAll => { PublishMessage(sendToAll.Path, sendToAll.Message, sendToAll.ExcludeSelf); }); Receive<Publish>(publish => { string path = Internal.Utils.MakeKey(Self.Path / Internal.Utils.EncodeName(publish.Topic)); if (publish.SendOneMessageToEachGroup) PublishToEachGroup(path, publish.Message); else PublishMessage(path, publish.Message); }); Receive<Put>(put => { if (put.Ref.Path.Address.HasGlobalScope) { Log.Warning("Registered actor must be local: [{0}]", put.Ref); } else { PutToRegistry(Internal.Utils.MakeKey(put.Ref), put.Ref); Context.Watch(put.Ref); } }); Receive<Remove>(remove => { Bucket bucket; if (_registry.TryGetValue(_cluster.SelfAddress, out bucket)) { ValueHolder valueHolder; if (bucket.Content.TryGetValue(remove.Path, out valueHolder) && valueHolder.Ref != null) { Context.Unwatch(valueHolder.Ref); PutToRegistry(remove.Path, null); } } }); Receive<Subscribe>(subscribe => { // each topic is managed by a child actor with the same name as the topic var encodedTopic = Internal.Utils.EncodeName(subscribe.Topic); _buffer.BufferOr(Internal.Utils.MakeKey(Self.Path / encodedTopic), subscribe, Sender, () => { var child = Context.Child(encodedTopic); if (!child.IsNobody()) { child.Forward(subscribe); } else { NewTopicActor(encodedTopic).Forward(subscribe); } }); }); Receive<RegisterTopic>(register => { HandleRegisterTopic(register.TopicRef); }); Receive<NoMoreSubscribers>(msg => { var key = Internal.Utils.MakeKey(Sender); _buffer.InitializeGrouping(key); Sender.Tell(TerminateRequest.Instance); }); Receive<NewSubscriberArrived>(msg => { var key = Internal.Utils.MakeKey(Sender); _buffer.ForwardMessages(key, Sender); }); Receive<GetTopics>(getTopics => { Sender.Tell(new CurrentTopics(GetCurrentTopics().ToArray())); }); Receive<Subscribed>(subscribed => { subscribed.Subscriber.Tell(subscribed.Ack); }); Receive<Unsubscribe>(unsubscribe => { var encodedTopic = Internal.Utils.EncodeName(unsubscribe.Topic); _buffer.BufferOr(Internal.Utils.MakeKey(Self.Path / encodedTopic), unsubscribe, Sender, () => { var child = Context.Child(encodedTopic); if (!child.IsNobody()) { child.Forward(unsubscribe); } else { // no such topic here } }); }); Receive<Unsubscribed>(unsubscribed => { unsubscribed.Subscriber.Tell(unsubscribed.Ack); }); Receive<Status>(status => { // only accept status from known nodes, otherwise old cluster with same address may interact // also accept from local for testing purposes if (_nodes.Contains(Sender.Path.Address) || Sender.Path.Address.HasLocalScope) { // gossip chat starts with a Status message, containing the bucket versions of the other node var delta = CollectDelta(status.Versions).ToArray(); if (delta.Length != 0) Sender.Tell(new Delta(delta)); if (!status.IsReplyToStatus && OtherHasNewerVersions(status.Versions)) Sender.Tell(new Status(versions: OwnVersions, isReplyToStatus: true)); // it will reply with Delta } }); Receive<Delta>(delta => { deltaCount += 1; // reply from Status message in the gossip chat // the Delta contains potential updates (newer versions) from the other node // only accept deltas/buckets from known nodes, otherwise there is a risk of // adding back entries when nodes are removed if (_nodes.Contains(Sender.Path.Address)) { foreach (var bucket in delta.Buckets) { if (_nodes.Contains(bucket.Owner)) { Bucket myBucket; if (!_registry.TryGetValue(bucket.Owner, out myBucket)) myBucket = new Bucket(bucket.Owner); if (bucket.Version > myBucket.Version) { _registry[bucket.Owner] = new Bucket(myBucket.Owner, bucket.Version, myBucket.Content.SetItems(bucket.Content)); } } } } }); Receive<GossipTick>(_ => HandleGossip()); Receive<Prune>(_ => HandlePrune()); Receive<Terminated>(terminated => { var key = Internal.Utils.MakeKey(terminated.ActorRef); Bucket bucket; if (_registry.TryGetValue(_cluster.SelfAddress, out bucket)) { ValueHolder holder; if (bucket.Content.TryGetValue(key, out holder) && terminated.ActorRef.Equals(holder.Ref)) { PutToRegistry(key, null); // remove } } _buffer.RecreateAndForwardMessagesIfNeeded(key, () => NewTopicActor(terminated.ActorRef.Path.Name)); }); Receive<ClusterEvent.CurrentClusterState>(state => { var nodes = state.Members .Where(m => m.Status != MemberStatus.Joining && IsMatchingRole(m)) .Select(m => m.Address); _nodes = new HashSet<Address>(nodes); }); Receive<ClusterEvent.MemberUp>(up => { if (IsMatchingRole(up.Member)) _nodes.Add(up.Member.Address); }); Receive<ClusterEvent.MemberLeft>(left => { if (IsMatchingRole(left.Member)) { _nodes.Remove(left.Member.Address); _registry.Remove(left.Member.Address); } }); Receive<ClusterEvent.MemberRemoved>(removed => { var member = removed.Member; if (member.Address == _cluster.SelfAddress) { Context.Stop(Self); } else if (IsMatchingRole(member)) { _nodes.Remove(member.Address); _registry.Remove(member.Address); } }); Receive<ClusterEvent.IMemberEvent>(_ => { /* ignore */ }); Receive<Count>(_ => { var count = _registry.Sum(entry => entry.Value.Content.Count(kv => kv.Value.Ref != null)); Sender.Tell(count); }); Receive<DeltaCount>(_ => { Sender.Tell(deltaCount); }); }