public void SimpleCacheMiss()
{
var bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(PooledBufferSize));
var dataAdapter = new TestCacheDataAdapter();
var cache = new PooledQueueCache(dataAdapter, NullLogger.Instance, null, null, TimeSpan.FromSeconds(10));
var evictionStrategy = new ChronologicalEvictionStrategy(NullLogger.Instance, new TimePurgePredicate(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)), null, null);
evictionStrategy.PurgeObservable = cache;
var converter = new CachedMessageConverter(bufferPool, evictionStrategy);
var seqNumber = 123;
var streamKey = Guid.NewGuid();
var stream = StreamId.Create(TestStreamNamespace, streamKey);
var cursor = cache.GetCursor(stream, new EventSequenceTokenV2(seqNumber));
// Start by enqueuing a message for stream, followed bu another one destined for another one
EnqueueMessage(streamKey);
EnqueueMessage(Guid.NewGuid());
// Consume the stream, should be fine
Assert.True(cache.TryGetNextMessage(cursor, out _));
Assert.False(cache.TryGetNextMessage(cursor, out _));
// Enqueue a new batch
// First and last messages destined for stream, following messages
// destined for other streams
EnqueueMessage(streamKey);
for (var idx = 0; idx < 20; idx++)
{
EnqueueMessage(Guid.NewGuid());
}
// Remove first three messages from the cache
cache.RemoveOldestMessage(); // Destined for stream, consumed
cache.RemoveOldestMessage(); // Not destined for stream
cache.RemoveOldestMessage(); // Destined for stream, not consumed
// Enqueue a new message for stream
EnqueueMessage(streamKey);
// Should throw since we missed the second message destined for stream
Assert.Throws <QueueCacheMissException>(() => cache.TryGetNextMessage(cursor, out _));
long EnqueueMessage(Guid streamId)
{
var now = DateTime.UtcNow;
var msg = new TestQueueMessage
{
StreamId = StreamId.Create(TestStreamNamespace, streamId),
SequenceNumber = seqNumber,
};
cache.Add(new List <CachedMessage>()
{
converter.ToCachedMessage(msg, now)
}, now);
seqNumber++;
return(msg.SequenceNumber);
}
}
public void AvoidCacheMissMultipleStreamsActive()
{
var bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(PooledBufferSize));
var dataAdapter = new TestCacheDataAdapter();
var cache = new PooledQueueCache(dataAdapter, NullLogger.Instance, null, null, TimeSpan.FromSeconds(30));
var evictionStrategy = new ChronologicalEvictionStrategy(NullLogger.Instance, new TimePurgePredicate(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)), null, null);
evictionStrategy.PurgeObservable = cache;
var converter = new CachedMessageConverter(bufferPool, evictionStrategy);
var seqNumber = 123;
var streamKey = Guid.NewGuid();
var stream = StreamId.Create(TestStreamNamespace, streamKey);
// Enqueue a message for our stream
var firstSequenceNumber = EnqueueMessage(streamKey);
// Enqueue a few other messages for other streams
EnqueueMessage(Guid.NewGuid());
EnqueueMessage(Guid.NewGuid());
// Consume the first event and see that the cursor has moved to last seen event (not matching our streamIdentity)
var cursor = cache.GetCursor(stream, new EventSequenceTokenV2(firstSequenceNumber));
Assert.True(cache.TryGetNextMessage(cursor, out var firstContainer));
Assert.False(cache.TryGetNextMessage(cursor, out _));
// Remove multiple events, including the one that the cursor is currently pointing to
cache.RemoveOldestMessage();
cache.RemoveOldestMessage();
cache.RemoveOldestMessage();
// Enqueue another message for stream
var lastSequenceNumber = EnqueueMessage(streamKey);
// Should be able to consume the event just pushed
Assert.True(cache.TryGetNextMessage(cursor, out var lastContainer));
Assert.Equal(stream, lastContainer.StreamId);
Assert.Equal(lastSequenceNumber, lastContainer.SequenceToken.SequenceNumber);
long EnqueueMessage(Guid streamId)
{
var now = DateTime.UtcNow;
var msg = new TestQueueMessage
{
StreamId = StreamId.Create(TestStreamNamespace, streamId),
SequenceNumber = seqNumber,
};
cache.Add(new List <CachedMessage>()
{
converter.ToCachedMessage(msg, now)
}, now);
seqNumber++;
return(msg.SequenceNumber);
}
}
public void GoldenPathTest()
{
var bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(PooledBufferSize));
var dataAdapter = new TestCacheDataAdapter();
var cache = new PooledQueueCache(dataAdapter, NullLogger.Instance, null, null);
var evictionStrategy = new ChronologicalEvictionStrategy(NullLogger.Instance, new TimePurgePredicate(TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(10)), null, null);
evictionStrategy.PurgeObservable = cache;
var converter = new CachedMessageConverter(bufferPool, evictionStrategy);
RunGoldenPath(cache, converter, 111);
}
public void SimpleCacheMiss()
{
var bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(PooledBufferSize));
var dataAdapter = new TestCacheDataAdapter();
var cache = new PooledQueueCache(dataAdapter, NullLogger.Instance, null, null, TimeSpan.FromSeconds(10));
var evictionStrategy = new ChronologicalEvictionStrategy(NullLogger.Instance, new TimePurgePredicate(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)), null, null);
evictionStrategy.PurgeObservable = cache;
var converter = new CachedMessageConverter(bufferPool, evictionStrategy);
int idx;
var seqNumber = 123;
var stream = StreamId.Create(TestStreamNamespace, Guid.NewGuid());
// First and last messages destined for stream, following messages
// destined for other streams
for (idx = 0; idx < 20; idx++)
{
var now = DateTime.UtcNow;
var msg = new TestQueueMessage
{
StreamId = (idx == 0) ? stream : StreamId.Create(TestStreamNamespace, Guid.NewGuid()),
SequenceNumber = seqNumber + idx,
};
cache.Add(new List <CachedMessage>()
{
converter.ToCachedMessage(msg, now)
}, now);
}
var cursor = cache.GetCursor(stream, new EventSequenceTokenV2(seqNumber));
// Remove first message
cache.RemoveOldestMessage();
// Enqueue a new message for stream
{
idx++;
var now = DateTime.UtcNow;
var msg = new TestQueueMessage
{
StreamId = stream,
SequenceNumber = seqNumber + idx,
};
cache.Add(new List <CachedMessage>()
{
converter.ToCachedMessage(msg, now)
}, now);
}
// Should throw since we missed the first message
Assert.Throws <QueueCacheMissException>(() => cache.TryGetNextMessage(cursor, out _));
}
/// <summary>
/// Default function to be called to create an EventhubQueueCache in IEventHubQueueCacheFactory.CreateCache method. User can
/// override this method to add more customization.
/// </summary>
protected virtual IEventHubQueueCache CreateCache(string partition, IEventHubDataAdapter dataAdatper, StreamStatisticOptions statisticOptions, IStreamQueueCheckpointer <string> checkpointer,
ILoggerFactory loggerFactory, IObjectPool <FixedSizeBuffer> bufferPool, string blockPoolId, TimePurgePredicate timePurge,
SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions, ITelemetryProducer telemetryProducer)
{
var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId);
var cacheMonitor = this.CacheMonitorFactory(cacheMonitorDimensions, loggerFactory, telemetryProducer);
var logger = loggerFactory.CreateLogger($"{typeof(EventHubQueueCache).FullName}.{sharedDimensions.EventHubPath}.{partition}");
var evictionStrategy = new ChronologicalEvictionStrategy(logger, timePurge, cacheMonitor, statisticOptions.StatisticMonitorWriteInterval);
return(new EventHubQueueCache(partition, EventHubAdapterReceiver.MaxMessagesPerRead, bufferPool, dataAdatper, evictionStrategy, checkpointer, logger,
cacheMonitor, statisticOptions.StatisticMonitorWriteInterval));
}
protected override IEventHubQueueCache CreateCache(string partition, IEventHubDataAdapter dataAdatper, StreamStatisticOptions options, IStreamQueueCheckpointer <string> checkpointer,
ILoggerFactory loggerFactory, IObjectPool <FixedSizeBuffer> bufferPool, string blockPoolId, TimePurgePredicate timePurge,
SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions, ITelemetryProducer telemetryProducer)
{
var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId);
var cacheMonitor = this.CacheMonitorFactory(cacheMonitorDimensions, loggerFactory, telemetryProducer);
var cacheLogger = loggerFactory.CreateLogger($"{typeof(EventHubQueueCache).FullName}.{this.name}.{partition}");
var evictionStrategy = new ChronologicalEvictionStrategy(cacheLogger, timePurge, cacheMonitor, options.StatisticMonitorWriteInterval);
//set defaultMaxAddCount to 10 so TryCalculateCachePressureContribution will start to calculate real contribution shortly
var cache = new QueueCacheForTesting(DefaultMaxAddCount, bufferPool, dataAdatper, evictionStrategy, checkpointer,
cacheLogger, cacheMonitor, options.StatisticMonitorWriteInterval);
this.caches.Add(cache);
return(cache);
}
private void AvoidCacheMiss(bool emptyCache)
{
var bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(PooledBufferSize));
var dataAdapter = new TestCacheDataAdapter();
var cache = new PooledQueueCache(dataAdapter, NullLogger.Instance, null, null, TimeSpan.FromSeconds(30));
var evictionStrategy = new ChronologicalEvictionStrategy(NullLogger.Instance, new TimePurgePredicate(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)), null, null);
evictionStrategy.PurgeObservable = cache;
var converter = new CachedMessageConverter(bufferPool, evictionStrategy);
var seqNumber = 123;
var stream = StreamId.Create(TestStreamNamespace, Guid.NewGuid());
// Enqueue a message for stream
var firstSequenceNumber = EnqueueMessage(stream);
// Consume first event
var cursor = cache.GetCursor(stream, new EventSequenceTokenV2(firstSequenceNumber));
Assert.True(cache.TryGetNextMessage(cursor, out var firstContainer));
Assert.Equal(stream, firstContainer.StreamId);
Assert.Equal(firstSequenceNumber, firstContainer.SequenceToken.SequenceNumber);
// Remove first message, that was consumed
cache.RemoveOldestMessage();
if (!emptyCache)
{
// Enqueue something not related to the stream
// so the cache isn't empty
EnqueueMessage(StreamId.Create(TestStreamNamespace, Guid.NewGuid()));
EnqueueMessage(StreamId.Create(TestStreamNamespace, Guid.NewGuid()));
EnqueueMessage(StreamId.Create(TestStreamNamespace, Guid.NewGuid()));
EnqueueMessage(StreamId.Create(TestStreamNamespace, Guid.NewGuid()));
EnqueueMessage(StreamId.Create(TestStreamNamespace, Guid.NewGuid()));
EnqueueMessage(StreamId.Create(TestStreamNamespace, Guid.NewGuid()));
}
// Enqueue another message for stream
var lastSequenceNumber = EnqueueMessage(stream);
// Should be able to consume the event just pushed
Assert.True(cache.TryGetNextMessage(cursor, out var lastContainer));
Assert.Equal(stream, lastContainer.StreamId);
Assert.Equal(lastSequenceNumber, lastContainer.SequenceToken.SequenceNumber);
long EnqueueMessage(StreamId streamId)
{
var now = DateTime.UtcNow;
var msg = new TestQueueMessage
{
StreamId = streamId,
SequenceNumber = seqNumber,
};
cache.Add(new List <CachedMessage>()
{
converter.ToCachedMessage(msg, now)
}, now);
seqNumber++;
return(msg.SequenceNumber);
}
}
