public KafkaEventConsumer(string groupId, IEnumerable <string> topics, string brokerList, IPubSubDispatcher <KafkaEventStream> dispatcher)
{
consumer = new Consumer <string, string>(constructConfig(groupId, brokerList, true), new StringDeserializer(System.Text.Encoding.UTF8), new StringDeserializer(System.Text.Encoding.UTF8));
this.topics = topics;
running = false;
this.dispatcher = dispatcher;
}
protected override Yield Stop(Result result)
{
// TODO (arnec): need to clean up upstream and downstream subscriptions
_dispatcher = null;
yield return(Coroutine.Invoke(base.Stop, new Result()));
result.Return();
}
public RabbitMQConsumer(IEnumerable <string> queueNames, IPubSubDispatcher <RabbitMQEventStream> dispatcher, string queueId)
{
this.queueNames = queueNames;
this.dispatcher = dispatcher;
this.queueId = queueId;
}
protected override Yield Stop(Result result)
{
// TODO (arnec): need to clean up upstream and downstream subscriptions
_dispatcher = null;
yield return Coroutine.Invoke(base.Stop, new Result());
result.Return();
}
//--- Methods ---
protected override Yield Start(XDoc config, IContainer container, Result result)
{
yield return Coroutine.Invoke(base.Start, config, container, new Result());
_log.DebugFormat("starting {0}", Self.Uri);
// make sure we have an IPubSubDispatcher registered
ContainerBuilder builder = null;
if(!container.IsRegistered<IPubSubDispatcher>()) {
builder = new ContainerBuilder();
builder.Register<Dispatcher>().As<IPubSubDispatcher>().ServiceScoped();
}
if(!container.IsRegistered<IPubSubDispatchQueueRepository>()) {
var localQueuePath = config["queue-path"].AsText;
builder = builder ?? new ContainerBuilder();
var retryTime = (config["failed-dispatch-retry"].AsInt ?? 60).Seconds();
if(string.IsNullOrEmpty(localQueuePath)) {
_log.Debug("no queue persistent path provided, using memory queues");
builder.Register(new MemoryPubSubDispatchQueueRepository(TimerFactory, retryTime))
.As<IPubSubDispatchQueueRepository>();
} else {
builder.Register(new PersistentPubSubDispatchQueueRepository(localQueuePath, TimerFactory, retryTime))
.As<IPubSubDispatchQueueRepository>();
}
}
if(builder != null) {
builder.Build(container);
}
// initialize dispatcher
_dispatcher = container.Resolve<IPubSubDispatcher>(
TypedParameter.From(new DispatcherConfig {
ServiceUri = Self,
ServiceAccessCookie = DreamCookie.NewSetCookie("service-key", InternalAccessKey, Self.Uri),
ServiceCookies = Cookies,
ServiceConfig = config
})
);
// check for upstream chaining
if(!config["upstream"].IsEmpty) {
_dispatcher.CombinedSet.AsDocument();
// we've been provided 1 or more upstream pubsub services that we need to subscribe to
foreach(XDoc upstream in config["upstream/uri"]) {
int retry = 0;
while(true) {
retry++;
_log.DebugFormat("setting up upstream chain to {0} (attempt {1})", upstream, retry);
XUri upstreamUri = upstream.AsUri;
// subscribe with an empty set, since there are no child subs at Start, but we need a place to subscribe updates on
XDoc emptySub = new XDoc("subscription-set").Elem("uri.owner", Self.Uri);
Result<DreamMessage> upstreamResult;
yield return upstreamResult = Plug.New(upstreamUri).Post(emptySub, new Result<DreamMessage>(TimeSpan.MaxValue));
if(upstreamResult.Value.IsSuccessful) {
XUri location = new XUri(upstreamResult.Value.Headers.Location).WithoutQuery();
string accessKey = upstreamResult.Value.ToDocument()["access-key"].AsText;
// subscribe the resulting location to our pubsub:///* changes
XDoc subscribeToChanges = new XDoc("subscription-set")
.Elem("uri.owner", upstreamUri.WithScheme("upstream"))
.Start("subscription")
.Attr("id", "1")
.Elem("channel", "pubsub://*/*")
.Add(DreamCookie.NewSetCookie("access-key", accessKey, location).AsSetCookieDocument)
.Start("recipient").Elem("uri", upstreamResult.Value.Headers.Location).End()
.End();
_dispatcher.RegisterSet(StringUtil.CreateAlphaNumericKey(8), subscribeToChanges, StringUtil.CreateAlphaNumericKey(8));
break;
}
_log.WarnFormat("unable to subscribe to upstream pubsub (attempt {0}): {1}", retry, upstreamResult.Value.Status);
if(retry >= 3) {
_log.WarnFormat("giving up on upstream chaining to {0}", upstream);
break;
}
yield return Async.Sleep(TimeSpan.FromMilliseconds(500));
continue;
}
}
}
if(!config["downstream"].IsEmpty) {
// we've been provided 1 or more downstream pubsub services that we need to get to subscribe to us
foreach(XDoc downstream in config["downstream/uri"]) {
int retry = 0;
while(true) {
retry++;
_log.DebugFormat("setting up downstream chain to {0} (attempt {1})", downstream, retry);
Result<DreamMessage> downstreamResult;
yield return downstreamResult = Plug.New(downstream.AsUri).Get(new Result<DreamMessage>(TimeSpan.MaxValue));
if(downstreamResult.Value.IsSuccessful) {
XDoc downstreamSet = downstreamResult.Value.ToDocument();
Tuplet<PubSubSubscriptionSet, bool> set = _dispatcher.RegisterSet(StringUtil.CreateAlphaNumericKey(8), downstreamSet, StringUtil.CreateAlphaNumericKey(8));
XUri locationUri = Self.At("subscribers", set.Item1.Location).Uri;
XUri featureUri = Self.At("subscribers").Uri;
_log.DebugFormat("downstream chain to {0} registered {1}", downstream, set.Item1.Location);
XDoc subscribeToChanges = new XDoc("subscription-set")
.Elem("uri.owner", Self.Uri)
.Start("subscription")
.Attr("id", "1")
.Elem("channel", "pubsub://*/*")
.Add(DreamCookie.NewSetCookie("access-key", set.Item1.AccessKey, featureUri).AsSetCookieDocument)
.Start("recipient").Elem("uri", locationUri).End()
.End();
yield return downstreamResult = Plug.New(downstream.AsUri).Post(subscribeToChanges, new Result<DreamMessage>(TimeSpan.MaxValue));
if(downstreamResult.Value.IsSuccessful) {
break;
}
_log.WarnFormat("unable to subscribe to downstream pubsub (attempt {0}): {1}", retry, downstreamResult.Value.Status);
} else {
_log.WarnFormat("unable to retrieve downstream set (attempt {0}): {1}", retry, downstreamResult.Value.Status);
}
if(retry >= 3) {
_log.WarnFormat("giving up on downstream chaining to {0}", downstream);
break;
}
yield return Async.Sleep(TimeSpan.FromMilliseconds(500));
}
}
}
result.Return();
}
public RedisConsumer(IEnumerable <string> channelNames, IPubSubDispatcher <RedisEventStream> dispatcher)
{
this.dispatcher = dispatcher;
this.channelNames = channelNames;
}
//--- Methods ---
protected override Yield Start(XDoc config, IContainer container, Result result)
{
yield return(Coroutine.Invoke(base.Start, config, container, new Result()));
_log.DebugFormat("starting {0}", Self.Uri);
// make sure we have an IPubSubDispatcher registered
if (!container.IsRegistered <IPubSubDispatcher>())
{
var builder = new ContainerBuilder();
builder.Register <Dispatcher>().As <IPubSubDispatcher>().ServiceScoped();
builder.Build(container);
}
// initialize dispatcher
_dispatcher = container.Resolve <IPubSubDispatcher>(TypedParameter.From(new DispatcherConfig {
ServiceUri = Self,
ServiceAccessCookie = DreamCookie.NewSetCookie("service-key", InternalAccessKey, Self.Uri),
ServiceCookies = Cookies,
ServiceConfig = config
}));
// check for upstream chaining
if (!config["upstream"].IsEmpty)
{
XDoc combinedset = _dispatcher.CombinedSet.AsDocument();
// we've been provided 1 or more upstream pubsub services that we need to subscribe to
foreach (XDoc upstream in config["upstream/uri"])
{
int retry = 0;
while (true)
{
retry++;
_log.DebugFormat("setting up upstream chain to {0} (attempt {1})", upstream, retry);
XUri upstreamUri = upstream.AsUri;
// subscribe with an empty set, since there are no child subs at Start, but we need a place to subscribe updates on
XDoc emptySub = new XDoc("subscription-set").Elem("uri.owner", Self.Uri);
Result <DreamMessage> upstreamResult;
yield return(upstreamResult = Plug.New(upstreamUri).Post(emptySub, new Result <DreamMessage>(TimeSpan.MaxValue)));
if (upstreamResult.Value.IsSuccessful)
{
XUri location = new XUri(upstreamResult.Value.Headers.Location).WithoutQuery();
string accessKey = upstreamResult.Value.ToDocument()["access-key"].AsText;
// subscribe the resulting location to our pubsub:///* changes
XDoc subscribeToChanges = new XDoc("subscription-set")
.Elem("uri.owner", upstreamUri.WithScheme("upstream"))
.Start("subscription")
.Attr("id", "1")
.Elem("channel", "pubsub://*/*")
.Add(DreamCookie.NewSetCookie("access-key", accessKey, location).AsSetCookieDocument)
.Start("recipient").Elem("uri", upstreamResult.Value.Headers.Location).End()
.End();
_dispatcher.RegisterSet(subscribeToChanges);
break;
}
_log.WarnFormat("unable to subscribe to upstream pubsub (attempt {0}): {1}", retry, upstreamResult.Value.Status);
if (retry >= 3)
{
_log.WarnFormat("giving up on upstream chaining to {0}", upstream);
break;
}
yield return(Async.Sleep(TimeSpan.FromMilliseconds(500)));
continue;
}
}
}
if (!config["downstream"].IsEmpty)
{
// we've been provided 1 or more downstream pubsub services that we need to get to subscribe to us
foreach (XDoc downstream in config["downstream/uri"])
{
int retry = 0;
while (true)
{
retry++;
_log.DebugFormat("setting up downstream chain to {0} (attempt {1})", downstream, retry);
Result <DreamMessage> downstreamResult;
yield return(downstreamResult = Plug.New(downstream.AsUri).Get(new Result <DreamMessage>(TimeSpan.MaxValue)));
if (downstreamResult.Value.IsSuccessful)
{
XDoc downstreamSet = downstreamResult.Value.ToDocument();
Tuplet <PubSubSubscriptionSet, bool> set = _dispatcher.RegisterSet(downstreamSet);
XUri locationUri = Self.At("subscribers", set.Item1.Location).Uri;
XUri featureUri = Self.At("subscribers").Uri;
_log.DebugFormat("downstream chain to {0} registered {1}", downstream, set.Item1.Location);
XDoc subscribeToChanges = new XDoc("subscription-set")
.Elem("uri.owner", Self.Uri)
.Start("subscription")
.Attr("id", "1")
.Elem("channel", "pubsub://*/*")
.Add(DreamCookie.NewSetCookie("access-key", set.Item1.AccessKey, featureUri).AsSetCookieDocument)
.Start("recipient").Elem("uri", locationUri).End()
.End();
yield return(downstreamResult = Plug.New(downstream.AsUri).Post(subscribeToChanges, new Result <DreamMessage>(TimeSpan.MaxValue)));
if (downstreamResult.Value.IsSuccessful)
{
break;
}
_log.WarnFormat("unable to subscribe to downstream pubsub (attempt {0}): {1}", retry, downstreamResult.Value.Status);
}
else
{
_log.WarnFormat("unable to retrieve downstream set (attempt {0}): {1}", retry, downstreamResult.Value.Status);
}
if (retry >= 3)
{
_log.WarnFormat("giving up on downstream chaining to {0}", downstream);
break;
}
yield return(Async.Sleep(TimeSpan.FromMilliseconds(500)));
}
}
}
result.Return();
}
public RedisConsumer(IEnumerable <string> topics, IPubSubDispatcher <RedisEventStream> dispatcher)
{
this.topics = topics;
this.dispatcher = dispatcher;
}
public ConsumerGroup(int consumerNum, string groupId, IEnumerable <string> topics, string brokerList, IPubSubDispatcher <KafkaEventStream> dispatcher)
{
for (int i = 0; i < consumerNum; i++)
{
var c = new KafkaEventConsumer(groupId, topics, brokerList, dispatcher);
consumers.Add(c);
}
}
