public void Can_roundtrip_DispatchItem_with_complex_event()
{
var body = new XDoc("msg").Elem("foo", "bar");
var channel = new XUri("channel://foo.com/bar");
var resource = new XUri("http://foo.com/baz/0");
var origin1 = new XUri("http://foo.com/baz/1");
var origin2 = new XUri("http://foo.com/baz/2");
var recipient1 = new DispatcherRecipient(new XUri("http://recipient1"));
var recipient2 = new DispatcherRecipient(new XUri("http://recipient2"));
var via1 = new XUri("http://via1");
var via2 = new XUri("http://via2");
var ev = new DispatcherEvent(body, channel, resource, origin1, origin2);
ev = ev.WithRecipient(false, recipient1, recipient2).WithVia(via1).WithVia(via2);
var item = new DispatchItem(
new XUri("http://foo"),
ev,
"abc"
);
var serializer = new DispatchItemSerializer();
var stream = serializer.ToStream(item);
var item2 = serializer.FromStream(stream);
Assert.AreEqual(item.Uri, item2.Uri, "uri mismatch");
Assert.AreEqual(item.Location, item2.Location, "location mismatch");
Assert.AreEqual(item.Event.Id, item2.Event.Id, "id mismatch");
Assert.AreEqual(body.ToCompactString(), item2.Event.AsDocument().ToCompactString(), "body mismatch");
Assert.AreEqual(channel, item2.Event.Channel, "channel mismatch");
Assert.AreEqual(resource, item2.Event.Resource, "resource mismatch");
Assert.AreEqual(origin1, item2.Event.Origins[0], "first origin mismatch");
Assert.AreEqual(origin2, item2.Event.Origins[1], "second origin mismatch");
Assert.AreEqual(recipient1.Uri, item2.Event.Recipients[0].Uri, "first recipient mismatch");
Assert.AreEqual(recipient2.Uri, item2.Event.Recipients[1].Uri, "second recipient mismatch");
Assert.AreEqual(via1, item2.Event.Via[0], "first via mismatch");
Assert.AreEqual(via2, item2.Event.Via[1], "second via mismatch");
}
public void Can_return_message_after_queue_has_been_disposed()
{
// Arrange
var item1 = new DispatchItem(new XUri("http://a"), new DispatcherEvent(new XDoc("msg"), new XUri("http://channl"), new XUri("http://resource")), "a");
var dispatchResult = new Result<bool>();
var dispatchQueue = new MemoryPubSubDispatchQueue("queue", TaskTimerFactory.Current, 1.Minutes(), (item) => dispatchResult);
dispatchQueue.Enqueue(item1);
// Act
dispatchQueue.Dispose();
dispatchResult.Return(true);
// Assert
// should not have thrown on the return, that is all
}
public void Can_roundtrip_DispatchItem()
{
var msg = new XDoc("msg");
var channel = new XUri("channel://foo.com/bar");
var origin = new XUri("http://foo.com/baz");
var ev = new DispatcherEvent(msg, channel, origin);
var item = new DispatchItem(
new XUri("http://foo"),
ev,
"abc"
);
var serializer = new DispatchItemSerializer();
var stream = serializer.ToStream(item);
var item2 = serializer.FromStream(stream);
Assert.AreEqual(item.Uri, item2.Uri, "uri mismatch");
Assert.AreEqual(item.Location, item2.Location, "location mismatch");
Assert.AreEqual(item.Event.Id, item2.Event.Id, "id mismatch");
}
public void Can_return_message_after_queue_has_been_disposed()
{
// Arrange
var queuePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
var item1 = new DispatchItem(new XUri("http://a"), new DispatcherEvent(new XDoc("msg"), new XUri("http://channl"), new XUri("http://resource")), "a");
var dispatchResult = new Result<bool>();
var dispatchQueue = new PersistentPubSubDispatchQueue(queuePath, TaskTimerFactory.Current, 1.Minutes(), i => dispatchResult);
dispatchQueue.Enqueue(item1);
// Act
dispatchQueue.Dispose();
dispatchResult.Return(true);
// Assert
// should not have thrown on the return, that is all
}
public void Can_dispatch_items()
{
// Arrange
var dispatched = new List<DispatchItem>();
Func<DispatchItem, Result<bool>> handler = (i) => {
dispatched.Add(i);
var result = new Result<bool>();
result.Return(true);
return result;
};
var dispatchQueue = new PersistentPubSubDispatchQueue(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()), TaskTimerFactory.Current, 1.Seconds(), handler);
var item1 = new DispatchItem(new XUri("http://a"), new DispatcherEvent(new XDoc("msg"), new XUri("http://channl"), new XUri("http://resource")), "a");
var item2 = new DispatchItem(new XUri("http://b"), new DispatcherEvent(new XDoc("msg"), new XUri("http://channl"), new XUri("http://resource")), "b");
// Act
dispatchQueue.Enqueue(item1);
dispatchQueue.Enqueue(item2);
// Assert
Assert.IsTrue(Wait.For(() => dispatched.Count == 2, 5.Seconds()), "items were not dispatched in time");
Assert.AreEqual(item1.Location, dispatched[0].Location, "wrong item location for first dispatched item");
Assert.AreEqual(item2.Location, dispatched[1].Location, "wrong item location for second dispatched item");
}
public void Speed()
{
var body = new XDoc("msg").Elem("foo", "bar");
var channel = new XUri("channel://foo.com/bar");
var resource = new XUri("http://foo.com/baz/0");
var origin1 = new XUri("http://foo.com/baz/1");
var origin2 = new XUri("http://foo.com/baz/2");
var recipient1 = new DispatcherRecipient(new XUri("http://recipient1"));
var recipient2 = new DispatcherRecipient(new XUri("http://recipient2"));
var via1 = new XUri("http://via1");
var via2 = new XUri("http://via2");
var ev = new DispatcherEvent(body, channel, resource, origin1, origin2);
ev = ev.WithRecipient(false, recipient1, recipient2).WithVia(via1).WithVia(via2);
var item = new DispatchItem(
new XUri("http://foo"),
ev,
"abc"
);
var serializer = new DispatchItemSerializer();
Stream stream = null;
var n = 100000;
var t = Stopwatch.StartNew();
for(var i = 0; i < n; i++) {
stream = serializer.ToStream(item);
}
t.Stop();
Console.WriteLine("serialize {0:0} items/sec", n / t.Elapsed.TotalSeconds);
t = Stopwatch.StartNew();
for(var i = 0; i < n; i++) {
serializer.FromStream(stream);
stream.Seek(0, SeekOrigin.Begin);
}
t.Stop();
Console.WriteLine("deserialize {0:0} items/sec", n / t.Elapsed.TotalSeconds);
}
//--- Methods ---
public void Enqueue(DispatchItem item)
{
EnsureNotDisposed();
_queue.Enqueue(item);
Kick();
}
public void ClearAndDisposed_queue_throws_on_access()
{
// Arrange
var queuePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
var dispatchQueue = new PersistentPubSubDispatchQueue(queuePath, TaskTimerFactory.Current, 1.Minutes(), i => new Result<bool>().WithReturn(true));
// Act
dispatchQueue.DeleteAndDispose();
// Assert
try {
var item = new DispatchItem(new XUri("http://a"), new DispatcherEvent(new XDoc("msg"), new XUri("http://channl"), new XUri("http://resource")), "a");
dispatchQueue.Enqueue(item);
Assert.Fail("Enqueue didn't throw");
} catch(ObjectDisposedException) {
} catch(AssertionException) {
throw;
} catch(Exception e) {
Assert.Fail(string.Format("Enqueue threw unexpected exception: {0}", e));
}
}
//--- Methods ---
public void Enqueue(DispatchItem item)
{
_dequeueHandler(item);
}
public void Creating_a_queue_with_persisted_items_starts_dispatch_immediately()
{
// Arrange
var queuePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
var item1 = new DispatchItem(new XUri("http://a"), new DispatcherEvent(new XDoc("msg"), new XUri("http://channl"), new XUri("http://resource")), "a");
var dispatchQueue = new PersistentPubSubDispatchQueue(queuePath, TaskTimerFactory.Current, 1.Seconds(), (item) => new Result<bool>().WithReturn(false));
dispatchQueue.Enqueue(item1);
dispatchQueue.Dispose();
var dispatched = new List<DispatchItem>();
Func<DispatchItem, Result<bool>> handler = (i) => {
dispatched.Add(i);
var result = new Result<bool>();
result.Return(true);
return result;
};
// Act
dispatchQueue = new PersistentPubSubDispatchQueue(queuePath, TaskTimerFactory.Current, 1.Seconds(), handler);
// Assert
Assert.IsTrue(Wait.For(() => dispatched.Count == 1, 5.Seconds()), "item was not dispatched in time");
Assert.AreEqual(item1.Location, dispatched[0].Location, "wrong item location for dispatched item");
}
public void ClearAndDispose_removes_queue_from_disk()
{
// Arrange
var queuePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
var item1 = new DispatchItem(new XUri("http://a"), new DispatcherEvent(new XDoc("msg"), new XUri("http://channl"), new XUri("http://resource")), "a");
var dispatchQueue = new PersistentPubSubDispatchQueue(queuePath, TaskTimerFactory.Current, 1.Minutes(), (item) => new Result<bool>().WithReturn(false));
dispatchQueue.Enqueue(item1);
Assert.IsTrue(Directory.GetFiles(queuePath).Length > 0, "queue directory did not contain any files");
// Act
dispatchQueue.DeleteAndDispose();
// Assert
Assert.IsFalse(Directory.Exists(queuePath), "queue directory still exists");
}
private Result<bool> TryDispatchItem(DispatchItem item)
{
var result = new Result<bool>();
_log.DebugFormat("dispatching event '{0}' to {1}", item.Event.Id, item.Uri);
Plug.New(item.Uri)
.WithCookieJar(_cookieJar)
.Post(item.Event.AsMessage(), new Result<DreamMessage>(TimeSpan.MaxValue))
.WhenDone(r => DispatchCompletion_Helper(item, r.Value, result));
return result;
}
private void DispatchCompletion_Helper(DispatchItem destination, DreamMessage response, Result<bool> result)
{
PubSubSubscriptionSet set;
lock(_subscriptionsByOwner) {
if(!_subscriptionByLocation.TryGetValue(destination.Location, out set)) {
_log.DebugFormat("the subscription at location '{0}' no longer exists, dropping event '{1}' ", destination.Location, destination.Event.Id);
result.Return(true);
return;
}
}
if(set.UsesFailureDuration) {
if(response.IsSuccessful || response.Status == DreamStatus.NotModified) {
result.Return(true);
return;
}
var queue = _queueRepository[set];
if(queue == null) {
_log.DebugFormat("the dispatch queue for subscription at location '{0}' no longer exists, dropping event '{1}' ", destination.Location, destination.Event.Id);
result.Return(true);
return;
}
if(queue.FailureWindow > set.MaxFailureDuration) {
_log.DebugFormat("the destination has failed continously for {0} and had an expiration failure window of {1}. The subscription at location '{2}' has been dropped, as has event '{3}'",
queue.FailureWindow,
set.MaxFailureDuration,
destination.Location,
destination.Event.Id
);
RemoveSet(set.Location);
result.Return(true);
return;
}
result.Return(false);
return;
}
if(response.IsSuccessful || response.Status == DreamStatus.NotModified) {
// if the post was a success, or didn't affect a change, clear any failure count
lock(_dispatchFailuresByLocation) {
if(_log.IsDebugEnabled) {
if(_dispatchFailuresByLocation.ContainsKey(destination.Location)) {
_log.Debug("zeroing out existing error count");
}
}
_dispatchFailuresByLocation.Remove(destination.Location);
}
} else {
// post was a failure, increase consecutive failures
if(_log.IsWarnEnabled) {
_log.WarnFormat("event dispatch to '{0}' failed: {1} - {2}", destination, response.Status, response.ToText());
}
lock(_dispatchFailuresByLocation) {
// NOTE (arnec): using ContainsKey instead of TryGetValue, since we're incrementing a value type in place
if(!_dispatchFailuresByLocation.ContainsKey(destination.Location)) {
_dispatchFailuresByLocation.Add(destination.Location, 1);
} else {
_dispatchFailuresByLocation[destination.Location]++;
}
var failures = _dispatchFailuresByLocation[destination.Location];
_log.DebugFormat("failure {0} out of {1} for set at location {2}", failures, set.MaxFailures, destination.Location);
// kick out a subscription set if one of its subscriptions fails too many times
if(failures > set.MaxFailures) {
_log.DebugFormat("exceeded max failures, kicking set at '{0}'", set.Location);
RemoveSet(destination.Location);
}
}
}
// Note (arnec): max-failure sets always "succeed" at dispatch since their queues are not kept around
result.Return(true);
}
private void TryDequeue()
{
_dequeueHandler(_currentItem).WhenDone(r => {
lock(_queue) {
if(_isDisposed) {
return;
}
if(r.HasException || !r.Value) {
if(_failureWindowStart == DateTime.MinValue) {
_failureWindowStart = DateTime.UtcNow;
}
_queueTimer.Change(_retryTime, TaskEnv.None);
return;
}
_failureWindowStart = DateTime.MinValue;
_queue.Dequeue();
_currentItem = null;
if(_queue.Count == 0) {
return;
}
_currentItem = _queue.Peek();
}
TryDequeue();
});
}
private void Kick()
{
if(_currentItem != null || _dequeueHandler == null) {
return;
}
lock(_queue) {
if(_currentItem != null) {
return;
}
if(_queue.Count == 0) {
return;
}
_currentItem = _queue.Peek();
Async.Fork(TryDequeue);
}
}
public void Disposed_queue_throws_on_access()
{
// Arrange
var dispatchQueue = new MemoryPubSubDispatchQueue("queue", TaskTimerFactory.Current, 1.Minutes(), i => new Result<bool>().WithReturn(true));
// Act
dispatchQueue.Dispose();
// Assert
try {
var item = new DispatchItem(new XUri("http://a"), new DispatcherEvent(new XDoc("msg"), new XUri("http://channl"), new XUri("http://resource")), "a");
dispatchQueue.Enqueue(item);
Assert.Fail("Enqueue didn't throw");
} catch(ObjectDisposedException) {
} catch(Exception e) {
Assert.Fail(string.Format("Enqueue threw unexpected exception: {0}", e));
}
}
public void Deserialize_wrong_version_throws()
{
var msg = new XDoc("msg");
var channel = new XUri("channel://foo.com/bar");
var origin = new XUri("http://foo.com/baz");
var ev = new DispatcherEvent(msg, channel, origin);
var item = new DispatchItem(
new XUri("http://foo"),
ev,
"abc"
);
var serializer = new DispatchItemSerializer();
var stream = serializer.ToStream(item);
stream.WriteByte(5);
stream.Position = 0;
try {
serializer.FromStream(stream);
Assert.Fail("should have thrown");
} catch(InvalidDataException) {
return;
}
}
public void Failed_dispatch_retries_after_sleep()
{
// Arrange
var item1 = new DispatchItem(new XUri("http://a"), new DispatcherEvent(new XDoc("msg"), new XUri("http://channl"), new XUri("http://resource")), "a");
var item2 = new DispatchItem(new XUri("http://b"), new DispatcherEvent(new XDoc("msg"), new XUri("http://channl"), new XUri("http://resource")), "b");
var dispatched = new List<Tuplet<DateTime, DispatchItem>>();
var dispatchCounter = 0;
Func<DispatchItem, Result<bool>> handler = (i) => {
dispatchCounter++;
dispatched.Add(new Tuplet<DateTime, DispatchItem>(DateTime.UtcNow, i));
var result = new Result<bool>();
result.Return(dispatchCounter > 2);
return result;
};
var dispatchQueue = new PersistentPubSubDispatchQueue(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()), TaskTimerFactory.Current, 1.Seconds(), handler);
// Act
dispatchQueue.Enqueue(item1);
dispatchQueue.Enqueue(item2);
// Assert
Assert.IsTrue(Wait.For(() => dispatched.Count >= 4, 10.Seconds()), "items were not dispatched in time");
Assert.AreEqual(item1.Location, dispatched[0].Item2.Location, "wrong item location for first failure item");
var dispatchTiming1 = dispatched[1].Item1 - dispatched[0].Item1;
Assert.IsTrue(dispatchTiming1 >= 1.Seconds(), "expected re-try in more than 1 second, was " + dispatchTiming1);
Assert.AreEqual(item1.Location, dispatched[1].Item2.Location, "wrong item location for second failure item");
var dispatchTiming2 = dispatched[2].Item1 - dispatched[1].Item1;
Assert.IsTrue(dispatchTiming2 >= 1.Seconds(), "expected re-try in more than 1 second, was " + dispatchTiming2);
Assert.AreEqual(item1.Location, dispatched[2].Item2.Location, "wrong item location for first success item");
var dispatchTiming3 = dispatched[3].Item1 - dispatched[2].Item1;
Assert.IsTrue(dispatchTiming3 < 1.Seconds(), "expected successful dispatch in less than 1 second, was " + dispatchTiming3);
Assert.AreEqual(item2.Location, dispatched[3].Item2.Location, "wrong item location for second success item");
}
public void Sets_loaded_by_initialize_use_provided_dequeue_handler()
{
// Arrange
Func<DispatchItem, Result<bool>> failHandler = (item) => new Result<bool>().WithReturn(false);
var dispatched = new List<DispatchItem>();
Func<DispatchItem, Result<bool>> successHandler = (item) => {
dispatched.Add(item);
return new Result<bool>().WithReturn(true);
};
_repository.InitializeRepository(failHandler);
var set = CreateSet();
_repository.RegisterOrUpdate(set);
var dispatchItem = new DispatchItem(new XUri("http://a"), new DispatcherEvent(new XDoc("msg"), new XUri("http://channl"), new XUri("http://resource")), "a");
_repository[set].Enqueue(dispatchItem);
_repository.Dispose();
CreateRepository();
// Act
_repository.InitializeRepository(successHandler);
// Assert
Assert.IsTrue(Wait.For(() => dispatched.Count > 0, 10.Seconds()), "no items were dispatched");
Assert.AreEqual(dispatchItem.Location, dispatched[0].Location, "wrong item location for first dispatched item");
}
public void Enqueue(DispatchItem item)
{
bool success;
do {
success = _dequeueHandler(item).Wait();
if(!success) {
FailureCount++;
}
} while(!success);
}