public void Can_aggregate_from_multiple_dream_hosts()
{
// set up hosts
_log.DebugFormat("---- creating upstream hosts");
var sourceHost1 = DreamTestHelper.CreateRandomPortHost();
var source1PubSub = Plug.New(sourceHost1.LocalHost.At("host", "$pubsub").With("apikey", sourceHost1.ApiKey));
var sourceHost2 = DreamTestHelper.CreateRandomPortHost();
var source2PubSub = Plug.New(sourceHost2.LocalHost.At("host", "$pubsub").With("apikey", sourceHost2.ApiKey));
// create aggregator
_log.DebugFormat("---- creating downstream host");
var aggregatorPath = "pubsubaggregator";
var aggregatorHost = DreamTestHelper.CreateRandomPortHost();
aggregatorHost.Host.RunScripts(new XDoc("config")
.Start("script").Start("action")
.Attr("verb", "POST")
.Attr("path", "/host/services")
.Start("config")
.Elem("path", aggregatorPath)
.Elem("sid", "sid://mindtouch.com/dream/2008/10/pubsub")
.Elem("apikey", "abc")
.Start("upstream")
.Elem("uri", source1PubSub.At("subscribers"))
.Elem("uri", source2PubSub.At("subscribers"))
.End()
.End().End(), null);
var aggregatorPubSub = aggregatorHost.LocalHost.At(aggregatorPath).With("apikey", "abc");
// create subscription
_log.DebugFormat("---- create downstream subscription");
var testUri = new XUri("http://mock/aggregator");
var serviceKey = "1234";
var accessCookie = DreamCookie.NewSetCookie("service-key", serviceKey, testUri);
var subscriberApiKey = "xyz";
var set = new XDoc("subscription-set")
.Elem("uri.owner", "http:///owner1")
.Start("subscription")
.Attr("id", "1")
.Add(accessCookie.AsSetCookieDocument)
.Elem("channel", "channel:///foo/*")
.Start("recipient")
.Attr("authtoken", subscriberApiKey)
.Elem("uri", testUri)
.End()
.End();
var r = aggregatorPubSub.At("subscribers").PostAsync(set).Wait();
Assert.IsTrue(r.IsSuccessful, r.Status.ToString());
Assert.AreEqual(DreamStatus.Created, r.Status);
// Verify that upstream host pubsub services have the subscription
Func<DreamMessage, bool> waitFunc = response => {
var sub = response.ToDocument()["subscription-set/subscription[channel='channel:///foo/*']"];
return (!sub.IsEmpty
&& sub["recipient/uri"].AsText.EqualsInvariantIgnoreCase(testUri.ToString())
&& sub["recipient/@authtoken"].AsText.EqualsInvariant(subscriberApiKey));
};
Assert.IsTrue(WaitFor(source1PubSub.At("diagnostics", "subscriptions"), waitFunc, TimeSpan.FromSeconds(5)), "source 1 didn't get the subscription");
Assert.IsTrue(WaitFor(source2PubSub.At("diagnostics", "subscriptions"), waitFunc, TimeSpan.FromSeconds(5)), "source 2 didn't get the subscription");
// set up destination mock
DispatcherEvent aggregatorEvent = new DispatcherEvent(
new XDoc("aggregator"),
new XUri("channel:///foo/bar"),
new XUri("http://foobar.com/some/page"));
DispatcherEvent source1Event = new DispatcherEvent(
new XDoc("source1"),
new XUri("channel:///foo/bar"),
new XUri("http://foobar.com/some/page"));
DispatcherEvent source2Event = new DispatcherEvent(
new XDoc("source2"),
new XUri("channel:///foo/bar"),
new XUri("http://foobar.com/some/page"));
var mock = MockPlug.Register(testUri);
// Publish event into aggregator
mock.Expect().Verb("POST").RequestDocument(aggregatorEvent.AsDocument());
r = aggregatorPubSub.At("publish").PostAsync(aggregatorEvent.AsMessage()).Wait();
Assert.IsTrue(r.IsSuccessful, r.Status.ToString());
Assert.IsTrue(mock.WaitAndVerify(TimeSpan.FromSeconds(10)), mock.VerificationFailure);
// Publish event into source1
mock.Reset();
mock.Expect().Verb("POST").RequestDocument(source1Event.AsDocument());
r = source1PubSub.At("publish").PostAsync(source1Event.AsMessage()).Wait();
Assert.IsTrue(r.IsSuccessful, r.Status.ToString());
Assert.IsTrue(mock.WaitAndVerify(TimeSpan.FromSeconds(10)), mock.VerificationFailure);
// Publish event into source2
mock.Reset();
mock.Expect().Verb("POST").RequestDocument(source2Event.AsDocument());
r = source2PubSub.At("publish").PostAsync(source2Event.AsMessage()).Wait();
Assert.IsTrue(r.IsSuccessful, r.Status.ToString());
Assert.IsTrue(mock.WaitAndVerify(TimeSpan.FromSeconds(10)), mock.VerificationFailure);
}
public void Parallel_chaining_subscription_and_message_propagation()
{
var rootPubsub = Plug.New(_hostInfo.Host.LocalMachineUri.At("host", "$pubsub", "subscribers"));
var goTrigger = new ManualResetEvent(false);
var pubsubResults = new List<Result<Plug>>();
for(var i = 0; i < 10; i++) {
pubsubResults.Add(Async.ForkThread(() => {
goTrigger.WaitOne();
return CreatePubSubService("upstream", new XDoc("config").Start("downstream").Elem("uri", rootPubsub).End()).WithInternalKey().AtLocalHost;
}, new Result<Plug>()));
}
var subscriberResults = new List<Result<Tuplet<XUri, AutoMockPlug>>>();
for(var i = 0; i < 20; i++) {
var mockUri = new XUri("http://mock/" + i);
subscriberResults.Add(Async.ForkThread(() => {
goTrigger.WaitOne();
rootPubsub.With("apikey", _hostInfo.ApiKey).Post(new XDoc("subscription-set")
.Elem("uri.owner", mockUri)
.Start("subscription")
.Attr("id", "1")
.Elem("channel", "channel://foo/bar")
.Start("recipient").Elem("uri", mockUri).End()
.End());
var mock = MockPlug.Register(mockUri);
return new Tuplet<XUri, AutoMockPlug>(mockUri, mock);
}, new Result<Tuplet<XUri, AutoMockPlug>>()));
}
goTrigger.Set();
var pubsubs = new List<Plug>();
foreach(var r in pubsubResults) {
pubsubs.Add(r.Wait());
}
var endpoints = new List<XUri>();
var mocks = new List<AutoMockPlug>();
foreach(var r in subscriberResults) {
var v = r.Wait();
endpoints.Add(v.Item1);
mocks.Add(v.Item2);
}
foreach(var pubsub in pubsubs) {
Plug plug = pubsub;
Wait.For(() => {
var set = plug.At("subscribers").Get();
return set.ToDocument()["subscription/recipient"].ListLength == endpoints.Count;
}, TimeSpan.FromSeconds(10));
}
var ev = new DispatcherEvent(new XDoc("blah"), new XUri("channel://foo/bar"), new XUri("http://foobar.com/some/page"));
foreach(var mock in mocks) {
mock.Expect().Verb("POST").RequestDocument(ev.AsDocument()).Response(DreamMessage.Ok());
}
pubsubs[0].At("publish").Post(ev.AsMessage());
foreach(var mock in mocks) {
Assert.IsTrue(mock.WaitAndVerify(TimeSpan.FromSeconds(10)), mock.VerificationFailure);
}
}
protected override Yield GetListenersByChannelResourceMatch(DispatcherEvent ev, Result<Dictionary<XUri, List<PubSubSubscription>>> result) {
if(!ev.Channel.Segments[1].EqualsInvariantIgnoreCase("pages")) {
// not a page DispatcherEvent or a page delete DispatcherEvent, use default matcher
Result<Dictionary<XUri, List<PubSubSubscription>>> baseResult;
yield return baseResult = Coroutine.Invoke(base.GetListenersByChannelResourceMatch, ev, new Result<Dictionary<XUri, List<PubSubSubscription>>>());
result.Return(baseResult);
yield break;
}
var matches = new List<PubSubSubscription>();
if(ev.Channel.Segments.Length <= 2 || !ev.Channel.Segments[2].EqualsInvariantIgnoreCase("delete")) {
// dispatch to all PubSubSubscriptions that listen for this DispatcherEvent and its contents
XDoc evDoc = ev.AsDocument();
uint? pageid = evDoc["pageid"].AsUInt;
string wikiId = evDoc["@wikiid"].AsText;
bool first = true;
_log.DebugFormat("trying dispatch based on channel & page PubSubSubscriptions for page '{0}' from wiki '{1}'", pageid, wikiId);
// fetch parent page id's for this page so that we can resolve infinite depth PubSubSubscriptions
Result<DreamMessage> pageHierarchyResult;
yield return pageHierarchyResult = _deki.At("pages", pageid.ToString()).WithHeader("X-Deki-Site", "id=" + wikiId).GetAsync();
DreamMessage pageHierarchy = pageHierarchyResult.Value;
if(pageHierarchy.IsSuccessful) {
XDoc pageDoc = pageHierarchy.ToDocument();
while(pageid.HasValue) {
List<Tuplet<PubSubSubscription, bool>> subs;
_subscriptionsByPage.TryGetValue(pageid.Value, out subs);
if(subs != null) {
// only the first pageId (the one from the event) triggers on non-infinite depth subs
foreach(var sub in subs) {
if((sub.Item2 || first) && !matches.Contains(sub.Item1)) {
matches.Add(sub.Item1);
}
}
}
// get parent id and then set pageDoc to the parent's subdoc, so we can descend the ancesstor tree further
pageid = pageDoc["page.parent/@id"].AsUInt;
pageDoc = pageDoc["page.parent"];
first = false;
}
} else {
_log.WarnFormat("unable to retrieve page doc for page '{0}': {1}", pageid, pageHierarchy.Status);
}
}
ICollection<PubSubSubscription> listeningSubs;
lock(_channelMap) {
// get all the PubSubSubscriptions that are wild card matches (which is basically those that didn't
// have any resources in their PubSubSubscription) and add them to the above matches
foreach(var sub in _resourceMap.GetMatches(new XUri("http://dummy/dummy"))) {
if(!matches.Contains(sub)) {
matches.Add(sub);
}
}
listeningSubs = _channelMap.GetMatches(ev.Channel, matches);
}
var listeners = new Dictionary<XUri, List<PubSubSubscription>>();
foreach(var sub in listeningSubs) {
List<PubSubSubscription> subs;
if(!listeners.TryGetValue(sub.Destination, out subs)) {
subs = new List<PubSubSubscription>();
listeners.Add(sub.Destination, subs);
subs.Add(sub);
} else if(!subs.Contains(sub)) {
subs.Add(sub);
}
}
result.Return(listeners);
yield break;
}
protected override Yield FilterRecipients(DispatcherEvent ev, PubSubSubscription subscription, Result<DispatcherEvent> result) {
var recipients2 = new List<DispatcherRecipient>();
uint? pageid = null;
string wikiId = null;
if(ev.HasDocument) {
var changeDoc = ev.AsDocument();
pageid = changeDoc["pageid"].AsUInt;
wikiId = changeDoc["@wikiid"].AsText;
}
var userIds = new Dictionary<int, DispatcherRecipient>();
foreach(var recipient in subscription.Recipients) {
var authtoken = recipient.Doc["@authtoken"].AsText;
if(string.IsNullOrEmpty(authtoken)) {
// if the recipient has no authtoken, but has a userid, collect the Id so we can authorize it against the page
int? userId = recipient.Doc["@userid"].AsInt;
if(userId.HasValue) {
userIds.Add(userId.Value, recipient);
}
} else if(authtoken == _authtoken) {
// master authtoken means the recipient doesn't need page level authorization (such as lucene)
recipients2.Add(recipient);
} else if(!string.IsNullOrEmpty(wikiId)) {
var key = authtoken + ":" + wikiId;
if(!_validatedKeys.Contains(key)) {
// no valid key found, need to check with API to validate
XDoc settings = null;
yield return _deki.At("site", "settings")
.With("apikey", _authtoken)
.WithHeader("X-Deki-Site", "id=" + wikiId)
.Get(new Result<DreamMessage>())
.Set(x => settings = x.IsSuccessful ? x.ToDocument() : null);
if(settings == null || !authtoken.EqualsInvariant(settings["security/api-key"].AsText)) {
continue;
}
_validatedKeys.Add(key);
}
// instance authtoken means the recipient doesn't need page level authorization (such as lucene)
recipients2.Add(recipient);
}
}
if(userIds.Count > 0 && (ev.Channel.Segments.Length <= 2 || !ev.Channel.Segments[2].EqualsInvariantIgnoreCase("delete"))) {
// check all userId's against the page to prune set to authorized users
var users = new XDoc("users");
foreach(int userid in userIds.Keys) {
users.Start("user").Attr("id", userid).End();
}
if(pageid.HasValue) {
Result<DreamMessage> userAuthResult;
yield return userAuthResult = _deki.At("pages", pageid.Value.ToString(), "allowed")
.With("permissions", "read,subscribe")
.With("filterdisabled", true)
.WithHeader("X-Deki-Site", "id=" + wikiId)
.PostAsync(users);
DreamMessage userAuth = userAuthResult.Value;
if(userAuth.IsSuccessful) {
int authorized = 0;
foreach(XDoc userid in userAuth.ToDocument()["user/@id"]) {
DispatcherRecipient recipient;
if(!userIds.TryGetValue(userid.AsInt.GetValueOrDefault(), out recipient)) {
continue;
}
authorized++;
recipients2.Add(recipient);
}
if(authorized != userIds.Count) {
_log.DebugFormat("requested auth on {0} users, received auth on {1} for page {2}", userIds.Count, authorized, pageid.Value);
}
} else {
_log.WarnFormat("unable to retrieve user auth for page '{0}': {1}", pageid, userAuth.Status);
}
}
}
result.Return(recipients2.Count == 0 ? null : ev.WithRecipient(true, recipients2.ToArray()));
yield break;
}
protected override Yield DetermineRecipients(DispatcherEvent ev, DispatcherRecipient[] recipients, Result<DispatcherEvent> result) {
List<DispatcherRecipient> recipients2 = new List<DispatcherRecipient>();
Dictionary<int, DispatcherRecipient> userIds = new Dictionary<int, DispatcherRecipient>();
foreach(DispatcherRecipient r in recipients) {
string authtoken = r.Doc["@authtoken"].AsText;
if(string.IsNullOrEmpty(authtoken)) {
// if the reciepient has no authtoken, but has a userid, collect the Id so we can authorize it against the page
int? userId = r.Doc["@userid"].AsInt;
if(userId.HasValue) {
userIds.Add(userId.Value, r);
}
} else if(authtoken == _authtoken) {
// authtoken means the recipient doesn't need page level authorization (such as lucene)
recipients2.Add(r);
}
}
if(userIds.Count > 0 && (ev.Channel.Segments.Length <= 2 || !StringUtil.EqualsInvariantIgnoreCase(ev.Channel.Segments[2], "delete"))) {
// check all userId's against the page to prune set to authorized users
XDoc users = new XDoc("users");
foreach(int userid in userIds.Keys) {
users.Start("user").Attr("id", userid).End();
}
XDoc changeDoc = ev.AsDocument();
uint? pageid = changeDoc["pageid"].AsUInt;
string wikiId = changeDoc["@wikiid"].AsText;
if(pageid.HasValue) {
Result<DreamMessage> userAuthResult;
yield return userAuthResult = _deki.At("pages", pageid.Value.ToString(), "allowed")
.With("permissions", "read,subscribe")
.With("filterdisabled", true)
.WithHeader("X-Deki-Site", "id=" + wikiId)
.PostAsync(users);
DreamMessage userAuth = userAuthResult.Value;
if(userAuth.IsSuccessful) {
int authorized = 0;
foreach(XDoc userid in userAuth.ToDocument()["user/@id"]) {
DispatcherRecipient recipient;
if(!userIds.TryGetValue(userid.AsInt.GetValueOrDefault(), out recipient)) {
continue;
}
authorized++;
recipients2.Add(recipient);
}
if(authorized != userIds.Count) {
_log.DebugFormat("requested auth on {0} users, received auth on {1} for page {2}", userIds.Count, authorized, pageid.Value);
}
} else {
_log.WarnFormat("unable to retrieve user auth for page '{0}': {1}", pageid, userAuth.Status);
}
}
}
result.Return(recipients2.Count == 0 ? null : ev.WithRecipient(true, recipients2.ToArray()));
yield break;
}
public void Dispatch_based_on_channel_match_with_different_wikiid_patterns_but_same_proxy_destination()
{
var ev = new DispatcherEvent(
new XDoc("msg"),
new XUri("event://sales.mindtouch.com/deki/comments/create"),
new XUri("http://foobar.com/some/comment"));
var dispatches = new List<DispatcherEvent>();
XUri testUri = new XUri("http://sales.mindtouch.com/").At(StringUtil.CreateAlphaNumericKey(4));
int dispatchCounter = 0;
MockPlug.Register(testUri, delegate(Plug plug, string verb, XUri uri, DreamMessage request, Result<DreamMessage> response) {
if(testUri == plug.Uri) {
lock(dispatches) {
dispatches.Add(new DispatcherEvent(request));
dispatchCounter++;
}
}
response.Return(DreamMessage.Ok());
});
int combinedSetUpdates = 0;
_dispatcher.CombinedSetUpdated += delegate {
combinedSetUpdates++;
_log.DebugFormat("combinedset updated ({0})", combinedSetUpdates);
};
var recipient1Uri = testUri.At("sub1");
_dispatcher.RegisterSet("abc",
new XDoc("subscription-set")
.Elem("uri.owner", "http:///owner1")
.Start("subscription")
.Attr("id", "1")
.Elem("channel", "event://sales.mindtouch.com/deki/comments/create")
.Elem("channel", "event://sales.mindtouch.com/deki/comments/update")
.Elem("uri.proxy", testUri)
.Start("recipient").Elem("uri", recipient1Uri).End()
.End(), "def");
var recipient2Uri = testUri.At("sub1");
_dispatcher.RegisterSet("qwe",
new XDoc("subscription-set")
.Elem("uri.owner", "http:///owner2")
.Start("subscription")
.Attr("id", "2")
.Elem("channel", "event://*/deki/comments/create")
.Elem("channel", "event://*/deki/comments/update")
.Elem("uri.proxy", testUri)
.Start("recipient").Elem("uri", recipient2Uri).End()
.End(), "asd");
// combinedset updates happen asynchronously, so give'em a chance
const int expectedCombinedSetUpdates = 2;
Assert.IsTrue(
Wait.For(() => combinedSetUpdates >= expectedCombinedSetUpdates, 10.Seconds()),
string.Format("expected at least {0} combined set updates, gave up after {1}", expectedCombinedSetUpdates, combinedSetUpdates)
);
const int expectedDispatches = 2;
_dispatcher.Dispatch(ev);
// dispatch happens async on a worker thread
Assert.IsTrue(
Wait.For(() => {
// Doing extra sleeping to improve the chance of catching excess dispatches
Thread.Sleep(100);
return dispatchCounter == expectedDispatches;
}, 10.Seconds()),
string.Format("expected at exactly {0} dispatches, gave up after {1}", expectedDispatches, dispatchCounter)
);
var sub1Event = dispatches.Where(x => x.Recipients.Any() && x.Recipients.FirstOrDefault().Uri == recipient1Uri).FirstOrDefault();
Assert.IsNotNull(sub1Event, "did not receive an event with recipient matching our first subscription");
Assert.AreEqual(ev.AsDocument(), sub1Event.AsDocument(), "event document is wrong");
Assert.AreEqual(ev.Id, sub1Event.Id, "event id is wrong");
var sub2Event = dispatches.Where(x => x.Recipients.Any() && x.Recipients.FirstOrDefault().Uri == recipient2Uri).FirstOrDefault();
Assert.IsNotNull(sub2Event, "did not receive an event with recipient matching our second subscription");
Assert.AreEqual(ev.AsDocument(), sub2Event.AsDocument(), "event document is wrong");
Assert.AreEqual(ev.Id, sub2Event.Id, "event id is wrong");
}