private StreamPosition GetStreamPosition(TestQueueMessage queueMessage)
{
IStreamIdentity streamIdentity = new StreamIdentity(queueMessage.StreamGuid, null);
StreamSequenceToken sequenceToken = queueMessage.SequenceToken;
return(new StreamPosition(streamIdentity, sequenceToken));
}
private ArraySegment <byte> SerializeMessageIntoPooledSegment(TestQueueMessage queueMessage)
{
// serialize payload
int size = SegmentBuilder.CalculateAppendSize(queueMessage.Data);
// get segment from current block
ArraySegment <byte> segment;
if (currentBuffer == null || !currentBuffer.TryGetSegment(size, out segment))
{
// no block or block full, get new block and try again
currentBuffer = bufferPool.Allocate();
//call EvictionStrategy's OnBlockAllocated method
this.evictionStrategy.OnBlockAllocated(currentBuffer);
// if this fails with clean block, then requested size is too big
if (!currentBuffer.TryGetSegment(size, out segment))
{
string errmsg = String.Format(CultureInfo.InvariantCulture,
"Message size is too big. MessageSize: {0}", size);
throw new ArgumentOutOfRangeException(nameof(queueMessage), errmsg);
}
}
// encode namespace, offset, partitionkey, properties and payload into segment
int writeOffset = 0;
SegmentBuilder.Append(segment, ref writeOffset, queueMessage.Data);
return(segment);
}
private StreamPosition GetStreamPosition(TestQueueMessage queueMessage)
{
var streamId = StreamId.Create(null, queueMessage.StreamGuid);
StreamSequenceToken sequenceToken = queueMessage.SequenceToken;
return(new StreamPosition(streamId, sequenceToken));
}
public void SimpleCacheMiss()
{
var bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(PooledBufferSize));
var dataAdapter = new TestCacheDataAdapter();
var cache = new PooledQueueCache(dataAdapter, NullLogger.Instance, null, null, TimeSpan.FromSeconds(10));
var evictionStrategy = new ChronologicalEvictionStrategy(NullLogger.Instance, new TimePurgePredicate(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)), null, null);
evictionStrategy.PurgeObservable = cache;
var converter = new CachedMessageConverter(bufferPool, evictionStrategy);
var seqNumber = 123;
var streamKey = Guid.NewGuid();
var stream = StreamId.Create(TestStreamNamespace, streamKey);
var cursor = cache.GetCursor(stream, new EventSequenceTokenV2(seqNumber));
// Start by enqueuing a message for stream, followed bu another one destined for another one
EnqueueMessage(streamKey);
EnqueueMessage(Guid.NewGuid());
// Consume the stream, should be fine
Assert.True(cache.TryGetNextMessage(cursor, out _));
Assert.False(cache.TryGetNextMessage(cursor, out _));
// Enqueue a new batch
// First and last messages destined for stream, following messages
// destined for other streams
EnqueueMessage(streamKey);
for (var idx = 0; idx < 20; idx++)
{
EnqueueMessage(Guid.NewGuid());
}
// Remove first three messages from the cache
cache.RemoveOldestMessage(); // Destined for stream, consumed
cache.RemoveOldestMessage(); // Not destined for stream
cache.RemoveOldestMessage(); // Destined for stream, not consumed
// Enqueue a new message for stream
EnqueueMessage(streamKey);
// Should throw since we missed the second message destined for stream
Assert.Throws <QueueCacheMissException>(() => cache.TryGetNextMessage(cursor, out _));
long EnqueueMessage(Guid streamId)
{
var now = DateTime.UtcNow;
var msg = new TestQueueMessage
{
StreamId = StreamId.Create(TestStreamNamespace, streamId),
SequenceNumber = seqNumber,
};
cache.Add(new List <CachedMessage>()
{
converter.ToCachedMessage(msg, now)
}, now);
seqNumber++;
return(msg.SequenceNumber);
}
}
public async Task EnqueueAbandonDequeue()
{
// create a test queue
Queue testQueue = await this.CreateQueue();
// send a test message
TestQueueMessage sendMessage = await this.SendMessage(testQueue);
// receive it
TestQueueMessage recvMessage = (TestQueueMessage)await testQueue.ReceiveAsync();
// compare it
Assert.IsTrue(sendMessage.Equals(recvMessage));
// abandon it
await testQueue.AbandonAsync(recvMessage);
// receive it again
TestQueueMessage recvMessage2 = (TestQueueMessage)await testQueue.ReceiveAsync();
// compare it
Assert.IsTrue(sendMessage.Equals(recvMessage2));
// delete message
await testQueue.CompleteAsync(recvMessage2);
// delete queue
await this.sb.DeleteQueueAsync(this.queueName);
}
private StreamPosition GetStreamPosition(TestQueueMessage queueMessage)
{
IStreamIdentity streamIdentity = new StreamIdentity(queueMessage.StreamGuid, queueMessage.StreamNamespace);
StreamSequenceToken sequenceToken = new EventSequenceTokenV2(queueMessage.SequenceNumber);
return(new StreamPosition(streamIdentity, sequenceToken));
}
public void NextInStreamTest()
{
IObjectPool <CachedMessageBlock <TestCachedMessage> > pool = new MyTestPooled();
ICacheDataAdapter <TestQueueMessage, TestCachedMessage> dataAdapter = new TestCacheDataAdapter();
CachedMessageBlock <TestCachedMessage> block = pool.Allocate();
int last = 0;
int sequenceNumber = 0;
// define 2 streams
var streams = new[] { new StreamIdentity(Guid.NewGuid(), null), new StreamIdentity(Guid.NewGuid(), null) };
// add both streams interleaved, until lock is full
while (block.HasCapacity)
{
var stream = streams[last % 2];
var message = new TestQueueMessage
{
StreamGuid = stream.Guid,
SequenceToken = new EventSequenceToken(sequenceNumber)
};
// add message to end of block
AddAndCheck(block, dataAdapter, message, 0, last - 1);
last++;
sequenceNumber += 2;
}
// get index of first stream
int streamIndex;
Assert.True(block.TryFindFirstMessage(streams[0], TestCacheDataComparer.Instance, out streamIndex));
Assert.Equal(0, streamIndex);
Assert.Equal(0, (block.GetSequenceToken(streamIndex, dataAdapter) as EventSequenceToken).SequenceNumber);
// find stream1 messages
int iteration = 1;
while (block.TryFindNextMessage(streamIndex + 1, streams[0], TestCacheDataComparer.Instance, out streamIndex))
{
Assert.Equal(iteration * 2, streamIndex);
Assert.Equal(iteration * 4, (block.GetSequenceToken(streamIndex, dataAdapter) as EventSequenceToken).SequenceNumber);
iteration++;
}
Assert.Equal(iteration, TestBlockSize / 2);
// get index of first stream
Assert.True(block.TryFindFirstMessage(streams[1], TestCacheDataComparer.Instance, out streamIndex));
Assert.Equal(1, streamIndex);
Assert.Equal(2, (block.GetSequenceToken(streamIndex, dataAdapter) as EventSequenceToken).SequenceNumber);
// find stream1 messages
iteration = 1;
while (block.TryFindNextMessage(streamIndex + 1, streams[1], TestCacheDataComparer.Instance, out streamIndex))
{
Assert.Equal(iteration * 2 + 1, streamIndex);
Assert.Equal(iteration * 4 + 2, (block.GetSequenceToken(streamIndex, dataAdapter) as EventSequenceToken).SequenceNumber);
iteration++;
}
Assert.Equal(iteration, TestBlockSize / 2);
}
public void AvoidCacheMissMultipleStreamsActive()
{
var bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(PooledBufferSize));
var dataAdapter = new TestCacheDataAdapter();
var cache = new PooledQueueCache(dataAdapter, NullLogger.Instance, null, null, TimeSpan.FromSeconds(30));
var evictionStrategy = new ChronologicalEvictionStrategy(NullLogger.Instance, new TimePurgePredicate(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)), null, null);
evictionStrategy.PurgeObservable = cache;
var converter = new CachedMessageConverter(bufferPool, evictionStrategy);
var seqNumber = 123;
var streamKey = Guid.NewGuid();
var stream = StreamId.Create(TestStreamNamespace, streamKey);
// Enqueue a message for our stream
var firstSequenceNumber = EnqueueMessage(streamKey);
// Enqueue a few other messages for other streams
EnqueueMessage(Guid.NewGuid());
EnqueueMessage(Guid.NewGuid());
// Consume the first event and see that the cursor has moved to last seen event (not matching our streamIdentity)
var cursor = cache.GetCursor(stream, new EventSequenceTokenV2(firstSequenceNumber));
Assert.True(cache.TryGetNextMessage(cursor, out var firstContainer));
Assert.False(cache.TryGetNextMessage(cursor, out _));
// Remove multiple events, including the one that the cursor is currently pointing to
cache.RemoveOldestMessage();
cache.RemoveOldestMessage();
cache.RemoveOldestMessage();
// Enqueue another message for stream
var lastSequenceNumber = EnqueueMessage(streamKey);
// Should be able to consume the event just pushed
Assert.True(cache.TryGetNextMessage(cursor, out var lastContainer));
Assert.Equal(stream, lastContainer.StreamId);
Assert.Equal(lastSequenceNumber, lastContainer.SequenceToken.SequenceNumber);
long EnqueueMessage(Guid streamId)
{
var now = DateTime.UtcNow;
var msg = new TestQueueMessage
{
StreamId = StreamId.Create(TestStreamNamespace, streamId),
SequenceNumber = seqNumber,
};
cache.Add(new List <CachedMessage>()
{
converter.ToCachedMessage(msg, now)
}, now);
seqNumber++;
return(msg.SequenceNumber);
}
}
private void AddAndCheck(CachedMessageBlock <TestCachedMessage> block, ICacheDataAdapter <TestQueueMessage, TestCachedMessage> dataAdapter, int first, int last, int sequenceNumber = 1)
{
var message = new TestQueueMessage
{
StreamGuid = StreamGuid,
SequenceToken = new EventSequenceToken(sequenceNumber)
};
AddAndCheck(block, dataAdapter, message, first, last);
}
private CachedMessage QueueMessageToCachedMessage(TestQueueMessage queueMessage, DateTime dequeueTimeUtc)
{
StreamPosition streamPosition = GetStreamPosition(queueMessage);
return(new CachedMessage
{
StreamGuid = streamPosition.StreamIdentity.Guid,
SequenceNumber = queueMessage.SequenceToken.SequenceNumber,
EventIndex = queueMessage.SequenceToken.EventIndex,
});
}
/// <summary>
/// Is this object equal?
/// </summary>
/// <param name="t">object to compare to</param>
/// <returns>true if equal, false otherwise</returns>
public bool Equals(TestQueueMessage t)
{
// If parameter is null return false:
if ((object)t == null)
{
return(false);
}
// Return true if the fields match:
return((this.TestString == t.TestString) && (this.TestInt == t.TestInt) && (this.TestBool == t.TestBool));
}
public void SimpleCacheMiss()
{
var bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(PooledBufferSize));
var dataAdapter = new TestCacheDataAdapter();
var cache = new PooledQueueCache(dataAdapter, NullLogger.Instance, null, null, TimeSpan.FromSeconds(10));
var evictionStrategy = new ChronologicalEvictionStrategy(NullLogger.Instance, new TimePurgePredicate(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)), null, null);
evictionStrategy.PurgeObservable = cache;
var converter = new CachedMessageConverter(bufferPool, evictionStrategy);
int idx;
var seqNumber = 123;
var stream = StreamId.Create(TestStreamNamespace, Guid.NewGuid());
// First and last messages destined for stream, following messages
// destined for other streams
for (idx = 0; idx < 20; idx++)
{
var now = DateTime.UtcNow;
var msg = new TestQueueMessage
{
StreamId = (idx == 0) ? stream : StreamId.Create(TestStreamNamespace, Guid.NewGuid()),
SequenceNumber = seqNumber + idx,
};
cache.Add(new List <CachedMessage>()
{
converter.ToCachedMessage(msg, now)
}, now);
}
var cursor = cache.GetCursor(stream, new EventSequenceTokenV2(seqNumber));
// Remove first message
cache.RemoveOldestMessage();
// Enqueue a new message for stream
{
idx++;
var now = DateTime.UtcNow;
var msg = new TestQueueMessage
{
StreamId = stream,
SequenceNumber = seqNumber + idx,
};
cache.Add(new List <CachedMessage>()
{
converter.ToCachedMessage(msg, now)
}, now);
}
// Should throw since we missed the first message
Assert.Throws <QueueCacheMissException>(() => cache.TryGetNextMessage(cursor, out _));
}
public CachedMessage ToCachedMessage(TestQueueMessage queueMessage, DateTime dequeueTimeUtc)
{
StreamPosition streamPosition = GetStreamPosition(queueMessage);
return(new CachedMessage
{
StreamId = streamPosition.StreamId,
SequenceNumber = queueMessage.SequenceNumber,
EnqueueTimeUtc = queueMessage.EnqueueTimeUtc,
DequeueTimeUtc = dequeueTimeUtc,
Segment = SerializeMessageIntoPooledSegment(queueMessage),
});
}
public CachedMessage ToCachedMessage(TestQueueMessage queueMessage, DateTime dequeueTimeUtc)
{
StreamPosition streamPosition = GetStreamPosition(queueMessage);
return(new CachedMessage
{
StreamGuid = streamPosition.StreamIdentity.Guid,
StreamNamespace = streamPosition.StreamIdentity.Namespace != null?string.Intern(streamPosition.StreamIdentity.Namespace) : null,
SequenceNumber = queueMessage.SequenceNumber,
EnqueueTimeUtc = queueMessage.EnqueueTimeUtc,
DequeueTimeUtc = dequeueTimeUtc,
Segment = SerializeMessageIntoPooledSegment(queueMessage),
});
}
/// <summary>
/// Is this object equal?
/// </summary>
/// <param name="obj">object to compare to</param>
/// <returns>true if equal, false otherwise</returns>
public override bool Equals(object obj)
{
// If parameter is null return false.
if (obj == null)
{
return(false);
}
// If parameter cannot be cast to TestQueueMessage return false.
TestQueueMessage t = obj as TestQueueMessage;
if ((object)t == null)
{
return(false);
}
// Return true if the fields match:
return((this.TestString == t.TestString) && (this.TestInt == t.TestInt) && (this.TestBool == t.TestBool));
}
public void TestInitialize()
{
// Arrange
var azureStorageQueueConfig = new AzureStorageQueueConfig
{
ConnectionString = AzureStorageData.ConnectionString,
QueueName = "osw-lib-dataaccess-azurestorage"
};
this.azureStorageQueue = AzureStorageQueueFactory.Create(
this.LoggerFactoryEnriched,
azureStorageQueueConfig);
this.azureStorageQueue.CreateIfNotExists();
this.testQueueMessage = new TestQueueMessage
{
Author = OrisicSoftworks
};
}
/// <summary>
/// Create and send a queue message
/// </summary>
/// <param name="queue">queue to send it to</param>
/// <returns>queue message that was sent</returns>
private async Task <TestQueueMessage> SendMessage(Queue queue)
{
// values to be used inside a queue message
string testMessageString = "sharad";
int testMessageInt = new Random().Next();
bool testMessageBool = true;
// create a test message
var m = new TestQueueMessage();
m.TestString = testMessageString;
m.TestInt = testMessageInt;
m.TestBool = testMessageBool;
// send the test message
await queue.SendAsync(m);
// return the message
return(m);
}
private void AddAndCheck(CachedMessageBlock <TestQueueMessage, TestCachedMessage> block, TestQueueMessage message, int first, int last)
{
Assert.AreEqual(first, block.OldestMessageIndex);
Assert.AreEqual(last, block.NewestMessageIndex);
Assert.IsTrue(block.HasCapacity);
block.Add(message);
last++;
Assert.AreEqual(first > last, block.IsEmpty);
Assert.AreEqual(last + 1 < TestBlockSize, block.HasCapacity);
Assert.AreEqual(first, block.OldestMessageIndex);
Assert.AreEqual(last, block.NewestMessageIndex);
Assert.IsTrue(block.GetSequenceToken(last).Equals(message.SequenceToken));
}
private void AvoidCacheMiss(bool emptyCache)
{
var bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(PooledBufferSize));
var dataAdapter = new TestCacheDataAdapter();
var cache = new PooledQueueCache(dataAdapter, NullLogger.Instance, null, null, TimeSpan.FromSeconds(30));
var evictionStrategy = new ChronologicalEvictionStrategy(NullLogger.Instance, new TimePurgePredicate(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)), null, null);
evictionStrategy.PurgeObservable = cache;
var converter = new CachedMessageConverter(bufferPool, evictionStrategy);
var seqNumber = 123;
var stream = StreamId.Create(TestStreamNamespace, Guid.NewGuid());
// Enqueue a message for stream
var firstSequenceNumber = EnqueueMessage(stream);
// Consume first event
var cursor = cache.GetCursor(stream, new EventSequenceTokenV2(firstSequenceNumber));
Assert.True(cache.TryGetNextMessage(cursor, out var firstContainer));
Assert.Equal(stream, firstContainer.StreamId);
Assert.Equal(firstSequenceNumber, firstContainer.SequenceToken.SequenceNumber);
// Remove first message, that was consumed
cache.RemoveOldestMessage();
if (!emptyCache)
{
// Enqueue something not related to the stream
// so the cache isn't empty
EnqueueMessage(StreamId.Create(TestStreamNamespace, Guid.NewGuid()));
EnqueueMessage(StreamId.Create(TestStreamNamespace, Guid.NewGuid()));
EnqueueMessage(StreamId.Create(TestStreamNamespace, Guid.NewGuid()));
EnqueueMessage(StreamId.Create(TestStreamNamespace, Guid.NewGuid()));
EnqueueMessage(StreamId.Create(TestStreamNamespace, Guid.NewGuid()));
EnqueueMessage(StreamId.Create(TestStreamNamespace, Guid.NewGuid()));
}
// Enqueue another message for stream
var lastSequenceNumber = EnqueueMessage(stream);
// Should be able to consume the event just pushed
Assert.True(cache.TryGetNextMessage(cursor, out var lastContainer));
Assert.Equal(stream, lastContainer.StreamId);
Assert.Equal(lastSequenceNumber, lastContainer.SequenceToken.SequenceNumber);
long EnqueueMessage(StreamId streamId)
{
var now = DateTime.UtcNow;
var msg = new TestQueueMessage
{
StreamId = streamId,
SequenceNumber = seqNumber,
};
cache.Add(new List <CachedMessage>()
{
converter.ToCachedMessage(msg, now)
}, now);
seqNumber++;
return(msg.SequenceNumber);
}
}
private void AddAndCheck(CachedMessageBlock <TestCachedMessage> block, ICacheDataAdapter <TestQueueMessage, TestCachedMessage> dataAdapter, TestQueueMessage message, int first, int last)
{
Assert.Equal(first, block.OldestMessageIndex);
Assert.Equal(last, block.NewestMessageIndex);
Assert.True(block.HasCapacity);
block.Add(message, DateTime.UtcNow, dataAdapter);
last++;
Assert.Equal(first > last, block.IsEmpty);
Assert.Equal(last + 1 < TestBlockSize, block.HasCapacity);
Assert.Equal(first, block.OldestMessageIndex);
Assert.Equal(last, block.NewestMessageIndex);
Assert.True(block.GetSequenceToken(last, dataAdapter).Equals(message.SequenceToken));
}
private StreamPosition GetStreamPosition(TestQueueMessage queueMessage)
{
StreamSequenceToken sequenceToken = new EventSequenceTokenV2(queueMessage.SequenceNumber);
return(new StreamPosition(queueMessage.StreamId, sequenceToken));
}