public async Task BasicTest()
{
ISequentialStore <Item> sequentialStore = await this.GetSequentialStore("basicTest");
var tasks = new List <Task>();
for (int i = 0; i < 10; i++)
{
int i1 = i;
tasks.Add(Task.Run(async() =>
{
for (int j = 0; j < 1000; j++)
{
long offset = await sequentialStore.Append(new Item {
Prop1 = i1 * 10 + j
});
Assert.True(offset <= 10000);
}
}));
}
await Task.WhenAll(tasks);
IEnumerable <(long offset, Item item)> batch = await sequentialStore.GetBatch(0, 10000);
IEnumerable <(long offset, Item item)> batchItems = batch as IList <(long offset, Item item)> ?? batch.ToList();
Assert.Equal(10000, batchItems.Count());
int counter = 0;
foreach ((long offset, Item item)batchItem in batchItems)
{
Assert.Equal(counter++, batchItem.offset);
}
}
public StoreTestResultCollection(ISequentialStore <T> store, int batchSize)
{
this.batchSize = batchSize;
this.store = store;
this.resultQueue = new Queue <T>();
this.lastLoadedPosition = -1;
}
public async Task <ISequentialStore <T> > GetSequentialStore <T>(string entityName)
{
IEntityStore <byte[], T> underlyingStore = this.GetEntityStore <byte[], T>(entityName);
ISequentialStore <T> sequentialStore = await SequentialStore <T> .Create(underlyingStore);
return(sequentialStore);
}
async Task <ISequentialStore <Item> > GetSequentialStore(string entityName)
{
IEntityStore <byte[], Item> entityStore = this.GetEntityStore <Item>(entityName);
ISequentialStore <Item> sequentialStore = await SequentialStore <Item> .Create(entityStore);
return(sequentialStore);
}
public async Task CreateWithOffsetTest()
{
IEntityStore <byte[], Item> entityStore = this.GetEntityStore <Item>($"testEntity{Guid.NewGuid().ToString()}");
ISequentialStore <Item> sequentialStore = await SequentialStore <Item> .Create(entityStore, 10150);
long offset = await sequentialStore.Append(new Item { Prop1 = 10 });
Assert.Equal(10150, offset);
sequentialStore = await SequentialStore <Item> .Create(entityStore);
for (int i = 0; i < 10; i++)
{
offset = await sequentialStore.Append(new Item { Prop1 = 20 });
Assert.Equal(10151 + i, offset);
}
sequentialStore = await SequentialStore <Item> .Create(entityStore);
offset = await sequentialStore.Append(new Item { Prop1 = 20 });
Assert.Equal(10161, offset);
IList <(long, Item)> batch = (await sequentialStore.GetBatch(0, 100)).ToList();
Assert.Equal(12, batch.Count);
long expectedOffset = 10150;
foreach ((long itemOffset, Item _) in batch)
{
Assert.Equal(itemOffset, expectedOffset++);
}
}
public async Task InitAsync(string storagePath, ISystemEnvironment systemEnvironment, bool optimizeForPerformance)
{
StoreProvider storeProvider;
try
{
var partitionsList = new List <string> {
"messages", "directMethods", "twins"
};
IDbStoreProvider dbStoreprovider = DbStoreProvider.Create(
new RocksDbOptionsProvider(systemEnvironment, optimizeForPerformance, Option.None <ulong>(), Option.None <int>(), Option.None <StorageLogLevel>()),
this.GetStoragePath(storagePath),
partitionsList);
storeProvider = new StoreProvider(dbStoreprovider);
}
catch (Exception ex) when(!ExceptionEx.IsFatal(ex))
{
Logger.LogError(ex, "Error creating RocksDB store. Falling back to in-memory store.");
storeProvider = new StoreProvider(new InMemoryDbStoreProvider());
}
this.messagesStore = await storeProvider.GetSequentialStore <MessageDetails>(MessageStorePartitionKey);
this.directMethodsStore = await storeProvider.GetSequentialStore <TestOperationResult>(DirectMethodsStorePartitionKey);
this.twinsStore = await storeProvider.GetSequentialStore <TestOperationResult>(TwinsStorePartitionKey);
}
public async Task RemoveTest()
{
ISequentialStore <Item> sequentialStore = await this.GetSequentialStore("removeTestEntity");
for (int i = 0; i < 10; i++)
{
long offset = await sequentialStore.Append(new Item { Prop1 = i });
Assert.Equal(i, offset);
}
List <(long offset, Item item)> batch = (await sequentialStore.GetBatch(0, 100)).ToList();
Assert.Equal(10, batch.Count);
int counter = 0;
foreach ((long offset, Item item)batchItem in batch)
{
Assert.Equal(counter++, batchItem.offset);
}
await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 0));
batch = (await sequentialStore.GetBatch(0, 100)).ToList();
Assert.Equal(9, batch.Count);
counter = 1;
foreach ((long offset, Item item)batchItem in batch)
{
Assert.Equal(counter++, batchItem.offset);
}
batch = (await sequentialStore.GetBatch(1, 100)).ToList();
Assert.Equal(9, batch.Count);
counter = 1;
foreach ((long offset, Item item)batchItem in batch)
{
Assert.Equal(counter++, batchItem.offset);
}
await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 0));
batch = (await sequentialStore.GetBatch(1, 100)).ToList();
Assert.Equal(9, batch.Count);
counter = 1;
foreach ((long offset, Item item)batchItem in batch)
{
Assert.Equal(counter++, batchItem.offset);
}
await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 1));
batch = (await sequentialStore.GetBatch(2, 100)).ToList();
Assert.Equal(8, batch.Count);
counter = 2;
foreach ((long offset, Item item)batchItem in batch)
{
Assert.Equal(counter++, batchItem.offset);
}
}
public MessageIterator(IKeyValueStore <string, MessageWrapper> entityStore,
ISequentialStore <MessageRef> endpointSequentialStore,
long startingOffset)
{
this.entityStore = entityStore;
this.endpointSequentialStore = endpointSequentialStore;
this.startingOffset = startingOffset < 0 ? 0 : startingOffset;
}
internal StoreTestResultCollectionEnumerator(ISequentialStore <T> store, int batchSize)
{
this.batchSize = batchSize;
this.store = store;
this.resultQueue = new Queue <T>();
this.lastLoadedPosition = -1;
this.current = default(T);
}
public async Task AddEndpoint(string endpointId)
{
ISequentialStore <MessageRef> sequentialStore = await this.storeProvider.GetSequentialStore <MessageRef>(endpointId);
if (this.endpointSequentialStores.TryAdd(endpointId, sequentialStore))
{
Events.SequentialStoreAdded(endpointId);
}
}
async Task <ISequentialStore <Item> > GetSequentialStore(string entityName, Option <long> defaultHeadOffset)
{
IEntityStore <byte[], Item> entityStore = this.GetEntityStore <Item>(entityName);
ISequentialStore <Item> sequentialStore = await defaultHeadOffset
.Map(d => SequentialStore <Item> .Create(entityStore, d))
.GetOrElse(() => SequentialStore <Item> .Create(entityStore));
return(sequentialStore);
}
public async Task AddEndpoint(string endpointId)
{
CheckpointData checkpointData = await this.checkpointStore.GetCheckpointDataAsync(endpointId, CancellationToken.None);
ISequentialStore <MessageRef> sequentialStore = await this.storeProvider.GetSequentialStore <MessageRef>(endpointId, checkpointData.Offset + 1);
if (this.endpointSequentialStores.TryAdd(endpointId, sequentialStore))
{
Events.SequentialStoreAdded(endpointId);
}
}
public async Task GetBatchTest(Option <long> defaultHeadOffset)
{
// Arrange
string entityId = $"invalidGetBatch{Guid.NewGuid().ToString()}";
long startOffset = defaultHeadOffset.GetOrElse(0);
ISequentialStore <Item> sequentialStore = await this.GetSequentialStore(entityId, defaultHeadOffset);
// Try to get the batch, should return empty batch.
List <(long, Item)> batch = (await sequentialStore.GetBatch(startOffset, 10)).ToList();
Assert.NotNull(batch);
Assert.Equal(0, batch.Count);
// Add 10 elements and remove 4, so that the range of elements is 4 - 9
for (int i = 0; i < 10; i++)
{
long offset = await sequentialStore.Append(new Item { Prop1 = i });
Assert.Equal(i + startOffset, offset);
}
for (int i = 0; i < 4; i++)
{
await sequentialStore.RemoveFirst((o, itm) => Task.FromResult(true));
}
// Try to get with starting offset <= 4, should return remaining elements - 4 - 9.
for (int i = 0; i <= 4; i++)
{
batch = (await sequentialStore.GetBatch(i + startOffset, 10)).ToList();
Assert.NotNull(batch);
Assert.Equal(10 - 4, batch.Count);
Assert.Equal(4 + startOffset, batch[0].Item1);
}
// Try to get with starting offset between 5 and 9, should return a valid batch.
for (int i = 5; i < 10; i++)
{
batch = (await sequentialStore.GetBatch(i + startOffset, 10)).ToList();
Assert.NotNull(batch);
Assert.Equal(10 - i, batch.Count);
Assert.Equal(i + startOffset, batch[0].Item1);
}
// Try to get with starting offset > 10, should return empty batch
for (int i = 10; i < 14; i++)
{
batch = (await sequentialStore.GetBatch(i + startOffset, 10)).ToList();
Assert.NotNull(batch);
Assert.Equal(0, batch.Count);
}
}
public async Task GetBatchTest()
{
// Arrange
ISequentialStore <Item> sequentialStore = await this.GetSequentialStore("invalidGetBatch");
// Try to get the batch, should return empty batch.
List <(long, Item)> batch = (await sequentialStore.GetBatch(0, 10)).ToList();
Assert.NotNull(batch);
Assert.Equal(0, batch.Count);
// Add 10 elements and remove 4, so that the range of elements is 4 - 9
for (int i = 0; i < 10; i++)
{
long offset = await sequentialStore.Append(new Item { Prop1 = i });
Assert.Equal(i, offset);
}
for (int i = 0; i < 4; i++)
{
await sequentialStore.RemoveFirst((o, itm) => Task.FromResult(true));
}
// Try to get with starting offset <= 4, should return remaining elements - 4 - 9.
for (int i = 0; i <= 4; i++)
{
batch = (await sequentialStore.GetBatch(i, 10)).ToList();
Assert.NotNull(batch);
Assert.Equal(10 - 4, batch.Count);
Assert.Equal(4, batch[0].Item1);
}
// Try to get with starting offset between 5 and 9, should return a valid batch.
for (int i = 5; i < 10; i++)
{
batch = (await sequentialStore.GetBatch(i, 10)).ToList();
Assert.NotNull(batch);
Assert.Equal(10 - i, batch.Count);
Assert.Equal(i, batch[0].Item1);
}
// Try to get with starting offset > 10, should return empty batch
for (int i = 10; i < 14; i++)
{
batch = (await sequentialStore.GetBatch(i, 10)).ToList();
Assert.NotNull(batch);
Assert.Equal(0, batch.Count);
}
}
public async Task CreateTest()
{
IEntityStore <byte[], Item> entityStore = this.GetEntityStore <Item>("testEntity");
ISequentialStore <Item> sequentialStore = await SequentialStore <Item> .Create(entityStore);
long offset = await sequentialStore.Append(new Item { Prop1 = 10 });
Assert.Equal(0, offset);
sequentialStore = await SequentialStore <Item> .Create(entityStore);
offset = await sequentialStore.Append(new Item { Prop1 = 20 });
Assert.Equal(1, offset);
}
public async Task CreateTest()
{
string entityName = $"testEntity{Guid.NewGuid().ToString()}";
IEntityStore <byte[], Item> entityStore = this.GetEntityStore <Item>(entityName);
ISequentialStore <Item> sequentialStore = await SequentialStore <Item> .Create(entityStore, entityName);
long offset = await sequentialStore.Append(new Item { Prop1 = 10 });
Assert.Equal(0, offset);
sequentialStore = await SequentialStore <Item> .Create(entityStore, entityName);
offset = await sequentialStore.Append(new Item { Prop1 = 20 });
Assert.Equal(1, offset);
}
private async Task ProcessAllAsync <T>(ISequentialStore <T> store, Action <T> callback)
{
int batchSize = 10;
long lastKey = 0;
var batch = await store.GetBatch(lastKey, batchSize);
while (batch.Any())
{
foreach ((long, T)item in batch)
{
lastKey = item.Item1;
callback(item.Item2);
}
batch = await store.GetBatch(lastKey + 1, batchSize);
}
}
public async Task BasicTest(Option <long> defaultHeadOffset)
{
string entityId = $"basicTest{Guid.NewGuid().ToString()}";
ISequentialStore <Item> sequentialStore = await this.GetSequentialStore(entityId, defaultHeadOffset);
long startOffset = defaultHeadOffset.GetOrElse(0);
var tasks = new List <Task>();
for (int i = 0; i < 10; i++)
{
int i1 = i;
tasks.Add(
Task.Run(
async() =>
{
for (int j = 0; j < 1000; j++)
{
long offset = await sequentialStore.Append(new Item {
Prop1 = i1 * 10 + j
});
Assert.True(offset <= 10000 + startOffset);
}
}));
}
await Task.WhenAll(tasks);
IEnumerable <(long offset, Item item)> batch = await sequentialStore.GetBatch(startOffset, 10000);
IEnumerable <(long offset, Item item)> batchItems = batch as IList <(long offset, Item item)> ?? batch.ToList();
Assert.Equal(10000, batchItems.Count());
long counter = startOffset;
foreach ((long offset, Item item)batchItem in batchItems)
{
Assert.Equal(counter++, batchItem.offset);
}
}
protected override void Load(ContainerBuilder builder)
{
// IMetricsScraper
builder.Register(c => new MetricsScraper(new string[] { "http://edgeHub:9600/metrics", "http://edgeAgent:9600/metrics" }))
.As <IMetricsScraper>()
.SingleInstance();
// Task<IMetricsStorage>
builder.Register(async c =>
{
IStoreProvider storeProvider = await c.Resolve <Task <IStoreProvider> >();
ISequentialStore <IEnumerable <Metric> > dataStore = await storeProvider.GetSequentialStore <IEnumerable <Metric> >("Metrics");
return(new MetricsStorage(dataStore) as IMetricsStorage);
})
.As <Task <IMetricsStorage> >()
.SingleInstance();
// IMetricsPublisher
builder.RegisterType <EdgeRuntimeDiagnosticsUpload>()
.As <IMetricsPublisher>()
.SingleInstance();
// Task<MetricsWorker>
builder.Register(async c =>
{
IMetricsScraper scraper = c.Resolve <IMetricsScraper>();
IMetricsPublisher publisher = c.Resolve <IMetricsPublisher>();
Task <IMetricsStorage> storage = c.Resolve <Task <IMetricsStorage> >();
return(new MetricsWorker(scraper, await storage, publisher));
})
.As <Task <MetricsWorker> >()
.SingleInstance();
base.Load(builder);
}
/// <summary>
/// Messages need to be cleaned up in the following scenarios –
/// 1.When the message expires (exceeds TTL)
/// 2.When a message has been processed (indicated by the the checkpoint)
/// 3.When there are 0 references to a message in the store from the message queues,
/// the message itself is deleted from the store (this means the message was successfully delivered to all endpoints).
/// // TODO - Update cleanup logic to cleanup expired messages from entity store as well.
/// </summary>
async Task CleanupMessages()
{
long totalCleanupCount = 0;
long totalCleanupStoreCount = 0;
try
{
while (true)
{
foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores)
{
try
{
if (this.cancellationTokenSource.IsCancellationRequested)
{
return;
}
Events.CleanupTaskStarted(endpointSequentialStore.Key);
CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(endpointSequentialStore.Key, CancellationToken.None);
ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value;
Events.CleanupCheckpointState(endpointSequentialStore.Key, checkpointData);
int cleanupEntityStoreCount = 0;
async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef)
{
if (checkpointData.Offset < offset &&
DateTime.UtcNow - messageRef.TimeStamp < messageRef.TimeToLive)
{
return(false);
}
bool deleteMessage = false;
// Decrement ref count.
await this.messageStore.messageEntityStore.Update(
messageRef.EdgeMessageId,
m =>
{
if (m.RefCount > 0)
{
m.RefCount--;
}
if (m.RefCount == 0)
{
deleteMessage = true;
}
return(m);
});
if (deleteMessage)
{
await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId);
cleanupEntityStoreCount++;
}
return(true);
}
// With the addition of PriorityQueues, the CleanupProcessor assumptions change slightly:
// Previously, we could always assume that if a message at the head of the queue should not be deleted,
// then none of the other messages in the queue should be either. Now, because we can have different TTL's
// for messages within the same queue, there can be messages that have expired in the queue after the head.
// This is okay because they will be cleaned up eventually and they will be ignored otherwise.
// TODO: Add optional CleanupProcessor mode that will go through the entire length of the queue each time to remove expired messages.
int cleanupCount = 0;
while (await sequentialStore.RemoveFirst(DeleteMessageCallback))
{
cleanupCount++;
}
totalCleanupCount += cleanupCount;
totalCleanupStoreCount += cleanupEntityStoreCount;
Events.CleanupCompleted(endpointSequentialStore.Key, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount);
await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token);
}
catch (Exception ex)
{
Events.ErrorCleaningMessagesForEndpoint(ex, endpointSequentialStore.Key);
}
}
await Task.Delay(this.GetCleanupTaskSleepTime());
}
}
catch (Exception ex)
{
Events.ErrorCleaningMessages(ex);
throw;
}
}
private async Task CleanQueue(bool checkEntireQueueOnCleanup)
{
long totalCleanupCount = 0;
long totalCleanupStoreCount = 0;
while (true)
{
foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores)
{
var messageQueueId = endpointSequentialStore.Key;
try
{
if (this.cancellationTokenSource.IsCancellationRequested)
{
return;
}
var(endpointId, priority) = MessageQueueIdHelper.ParseMessageQueueId(messageQueueId);
Events.CleanupTaskStarted(messageQueueId);
CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(messageQueueId, CancellationToken.None);
ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value;
Events.CleanupCheckpointState(messageQueueId, checkpointData);
int cleanupEntityStoreCount = 0;
// If checkEntireQueueOnCleanup is set to false, we only peek the head, message counts is tailOffset-headOffset+1
// otherwise count while iterating over the queue.
var headOffset = 0L;
var tailOffset = sequentialStore.GetTailOffset(CancellationToken.None);
var messageCount = 0L;
async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef)
{
var expiry = messageRef.TimeStamp + messageRef.TimeToLive;
if (offset > checkpointData.Offset && expiry > DateTime.UtcNow)
{
// message is not sent and not expired, increase message counts
messageCount++;
return(false);
}
headOffset = Math.Max(headOffset, offset);
bool deleteMessage = false;
// Decrement ref count.
var message = await this.messageStore.messageEntityStore.Update(
messageRef.EdgeMessageId,
m =>
{
if (m.RefCount > 0)
{
m.RefCount--;
}
if (m.RefCount == 0)
{
deleteMessage = true;
}
return(m);
});
if (deleteMessage)
{
if (offset > checkpointData.Offset && expiry <= DateTime.UtcNow)
{
this.expiredCounter.Increment(1, new[] { "ttl_expiry", message?.Message.GetSenderId(), message?.Message.GetOutput(), bool.TrueString });
}
await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId);
cleanupEntityStoreCount++;
}
return(true);
}
// With the addition of PriorityQueues, the CleanupProcessor assumptions change slightly:
// Previously, we could always assume that if a message at the head of the queue should not be deleted,
// then none of the other messages in the queue should be either. Now, because we can have different TTL's
// for messages within the same queue, there can be messages that have expired in the queue after the head.
// The checkEntireQueueOnCleanup flag is an environment variable for edgeHub. If it is set to true, we will
// check the entire queue every time cleanup processor runs. If it is set to false, we just remove the oldest
// items in the queue until we get to one that is not expired.
int cleanupCount = 0;
if (checkEntireQueueOnCleanup)
{
IEnumerable <(long, MessageRef)> batch;
long offset = sequentialStore.GetHeadOffset(this.cancellationTokenSource.Token);
do
{
batch = await sequentialStore.GetBatch(offset, CleanupBatchSize);
foreach ((long, MessageRef)messageWithOffset in batch)
{
if (await sequentialStore.RemoveOffset(DeleteMessageCallback, messageWithOffset.Item1, this.cancellationTokenSource.Token))
{
cleanupCount++;
}
}
offset += CleanupBatchSize;
}while (batch.Any());
}
else
{
while (await sequentialStore.RemoveFirst(DeleteMessageCallback))
{
cleanupCount++;
}
messageCount = tailOffset - headOffset + 1;
}
// update Metrics for message counts
Checkpointer.Metrics.QueueLength.Set(messageCount, new[] { endpointId, priority.ToString(), bool.TrueString });
totalCleanupCount += cleanupCount;
totalCleanupStoreCount += cleanupEntityStoreCount;
Events.CleanupCompleted(messageQueueId, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount);
await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token);
}
catch (Exception ex)
{
Events.ErrorCleaningMessagesForEndpoint(ex, messageQueueId);
}
}
await Task.Delay(this.GetCleanupTaskSleepTime());
}
}
public Task RemoveStore <T>(ISequentialStore <T> sequentialStore)
{
this.dbStoreProvider.RemoveDbStore(sequentialStore.EntityName);
return(Task.CompletedTask);
}
private async Task CleanQueue(bool checkEntireQueueOnCleanup)
{
long totalCleanupCount = 0;
long totalCleanupStoreCount = 0;
while (true)
{
foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores)
{
try
{
if (this.cancellationTokenSource.IsCancellationRequested)
{
return;
}
Events.CleanupTaskStarted(endpointSequentialStore.Key);
CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(endpointSequentialStore.Key, CancellationToken.None);
ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value;
Events.CleanupCheckpointState(endpointSequentialStore.Key, checkpointData);
int cleanupEntityStoreCount = 0;
async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef)
{
if (checkpointData.Offset < offset &&
DateTime.UtcNow - messageRef.TimeStamp < messageRef.TimeToLive)
{
return(false);
}
bool deleteMessage = false;
// Decrement ref count.
await this.messageStore.messageEntityStore.Update(
messageRef.EdgeMessageId,
m =>
{
if (m.RefCount > 0)
{
m.RefCount--;
}
if (m.RefCount == 0)
{
deleteMessage = true;
}
return(m);
});
if (deleteMessage)
{
await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId);
cleanupEntityStoreCount++;
}
return(true);
}
// With the addition of PriorityQueues, the CleanupProcessor assumptions change slightly:
// Previously, we could always assume that if a message at the head of the queue should not be deleted,
// then none of the other messages in the queue should be either. Now, because we can have different TTL's
// for messages within the same queue, there can be messages that have expired in the queue after the head.
// The checkEntireQueueOnCleanup flag is an environment variable for edgeHub. If it is set to true, we will
// check the entire queue every time cleanup processor runs. If it is set to false, we just remove the oldest
// items in the queue until we get to one that is not expired.
int cleanupCount = 0;
if (checkEntireQueueOnCleanup)
{
IEnumerable <(long, MessageRef)> batch;
long offset = sequentialStore.GetHeadOffset(this.cancellationTokenSource.Token);
do
{
batch = await sequentialStore.GetBatch(offset, CleanupBatchSize);
foreach ((long, MessageRef)messageWithOffset in batch)
{
if (await sequentialStore.RemoveOffset(DeleteMessageCallback, messageWithOffset.Item1, this.cancellationTokenSource.Token))
{
cleanupCount++;
}
}
offset = offset + CleanupBatchSize;
}while (batch.Any());
}
else
{
while (await sequentialStore.RemoveFirst(DeleteMessageCallback))
{
cleanupCount++;
}
}
totalCleanupCount += cleanupCount;
totalCleanupStoreCount += cleanupEntityStoreCount;
Events.CleanupCompleted(endpointSequentialStore.Key, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount);
await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token);
}
catch (Exception ex)
{
Events.ErrorCleaningMessagesForEndpoint(ex, endpointSequentialStore.Key);
}
}
await Task.Delay(this.GetCleanupTaskSleepTime());
}
}
public StoreTestResultCollection(ISequentialStore <T> store, int batchSize)
{
this.enumerator = new StoreTestResultCollectionEnumerator(store, batchSize);
}
public MetricsStorage(ISequentialStore <IEnumerable <Metric> > sequentialStore, TimeSpan maxMetricAge)
{
this.dataStore = Preconditions.CheckNotNull(sequentialStore, nameof(sequentialStore));
this.maxMetricAge = Preconditions.CheckNotNull(maxMetricAge, nameof(maxMetricAge));
}
/// <summary>
/// Messages need to be cleaned up in the following scenarios –
/// 1. When the message expires (exceeds TTL)
/// 2. When a message has been processed (indicated by the the checkpoint)
/// 3. When there are 0 references to a message in the store from the message queues,
/// the message itself is deleted from the store (this means the message was successfully delivered to all endpoints).
/// // TODO - Update cleanup logic to cleanup expired messages from entity store as well.
/// </summary>
async Task CleanupMessages()
{
long totalCleanupCount = 0;
long totalCleanupStoreCount = 0;
try
{
TimeSpan cleanupTaskSleepTime = this.messageStore.timeToLive.TotalSeconds / 2 < CleanupTaskFrequency.TotalSeconds
? TimeSpan.FromSeconds(this.messageStore.timeToLive.TotalSeconds / 2)
: CleanupTaskFrequency;
while (true)
{
foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores)
{
try
{
if (this.cancellationTokenSource.IsCancellationRequested)
{
return;
}
Events.CleanupTaskStarted(endpointSequentialStore.Key);
CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(endpointSequentialStore.Key, CancellationToken.None);
ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value;
Events.CleanupCheckpointState(endpointSequentialStore.Key, checkpointData);
int cleanupEntityStoreCount = 0;
async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef)
{
if (checkpointData.Offset < offset &&
DateTime.UtcNow - messageRef.TimeStamp < this.messageStore.timeToLive)
{
return(false);
}
bool deleteMessage = false;
// Decrement ref count.
await this.messageStore.messageEntityStore.Update(
messageRef.EdgeMessageId,
m =>
{
if (m.RefCount > 0)
{
m.RefCount--;
}
if (m.RefCount == 0)
{
deleteMessage = true;
}
return(m);
});
if (deleteMessage)
{
await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId);
cleanupEntityStoreCount++;
}
return(true);
}
int cleanupCount = 0;
while (await sequentialStore.RemoveFirst(DeleteMessageCallback))
{
cleanupCount++;
}
totalCleanupCount += cleanupCount;
totalCleanupStoreCount += cleanupEntityStoreCount;
Events.CleanupCompleted(endpointSequentialStore.Key, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount);
await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token);
}
catch (Exception ex)
{
Events.ErrorCleaningMessagesForEndpoint(ex, endpointSequentialStore.Key);
}
}
await Task.Delay(cleanupTaskSleepTime);
}
}
catch (Exception ex)
{
Events.ErrorCleaningMessages(ex);
throw;
}
}
public async Task RemoveTest(Option <long> defaultHeadOffset)
{
string entityId = $"removeTestEntity{Guid.NewGuid().ToString()}";
long startOffset = defaultHeadOffset.GetOrElse(0);
ISequentialStore <Item> sequentialStore = await this.GetSequentialStore(entityId, defaultHeadOffset);
for (int i = 0; i < 10; i++)
{
long offset = await sequentialStore.Append(new Item { Prop1 = i });
Assert.Equal(i + startOffset, offset);
}
List <(long offset, Item item)> batch = (await sequentialStore.GetBatch(startOffset, 100)).ToList();
Assert.Equal(10, batch.Count);
long counter = startOffset;
foreach ((long offset, Item item)batchItem in batch)
{
Assert.Equal(counter++, batchItem.offset);
}
await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 0));
batch = (await sequentialStore.GetBatch(startOffset, 100)).ToList();
Assert.Equal(9, batch.Count);
counter = startOffset + 1;
foreach ((long offset, Item item)batchItem in batch)
{
Assert.Equal(counter++, batchItem.offset);
}
batch = (await sequentialStore.GetBatch(startOffset + 1, 100)).ToList();
Assert.Equal(9, batch.Count);
counter = startOffset + 1;
foreach ((long offset, Item item)batchItem in batch)
{
Assert.Equal(counter++, batchItem.offset);
}
await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 0));
batch = (await sequentialStore.GetBatch(startOffset + 1, 100)).ToList();
Assert.Equal(9, batch.Count);
counter = startOffset + 1;
foreach ((long offset, Item item)batchItem in batch)
{
Assert.Equal(counter++, batchItem.offset);
}
await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 1));
batch = (await sequentialStore.GetBatch(startOffset + 2, 100)).ToList();
Assert.Equal(8, batch.Count);
counter = startOffset + 2;
foreach ((long offset, Item item)batchItem in batch)
{
Assert.Equal(counter++, batchItem.offset);
}
}
