public async Task RemoveTest() { ISequentialStore <Item> sequentialStore = await this.GetSequentialStore("removeTestEntity"); for (int i = 0; i < 10; i++) { long offset = await sequentialStore.Append(new Item { Prop1 = i }); Assert.Equal(i, offset); } List <(long offset, Item item)> batch = (await sequentialStore.GetBatch(0, 100)).ToList(); Assert.Equal(10, batch.Count); int counter = 0; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 0)); batch = (await sequentialStore.GetBatch(0, 100)).ToList(); Assert.Equal(9, batch.Count); counter = 1; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } batch = (await sequentialStore.GetBatch(1, 100)).ToList(); Assert.Equal(9, batch.Count); counter = 1; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 0)); batch = (await sequentialStore.GetBatch(1, 100)).ToList(); Assert.Equal(9, batch.Count); counter = 1; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 1)); batch = (await sequentialStore.GetBatch(2, 100)).ToList(); Assert.Equal(8, batch.Count); counter = 2; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } }
public async Task GetBatchTest(Option <long> defaultHeadOffset) { // Arrange string entityId = $"invalidGetBatch{Guid.NewGuid().ToString()}"; long startOffset = defaultHeadOffset.GetOrElse(0); ISequentialStore <Item> sequentialStore = await this.GetSequentialStore(entityId, defaultHeadOffset); // Try to get the batch, should return empty batch. List <(long, Item)> batch = (await sequentialStore.GetBatch(startOffset, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(0, batch.Count); // Add 10 elements and remove 4, so that the range of elements is 4 - 9 for (int i = 0; i < 10; i++) { long offset = await sequentialStore.Append(new Item { Prop1 = i }); Assert.Equal(i + startOffset, offset); } for (int i = 0; i < 4; i++) { await sequentialStore.RemoveFirst((o, itm) => Task.FromResult(true)); } // Try to get with starting offset <= 4, should return remaining elements - 4 - 9. for (int i = 0; i <= 4; i++) { batch = (await sequentialStore.GetBatch(i + startOffset, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(10 - 4, batch.Count); Assert.Equal(4 + startOffset, batch[0].Item1); } // Try to get with starting offset between 5 and 9, should return a valid batch. for (int i = 5; i < 10; i++) { batch = (await sequentialStore.GetBatch(i + startOffset, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(10 - i, batch.Count); Assert.Equal(i + startOffset, batch[0].Item1); } // Try to get with starting offset > 10, should return empty batch for (int i = 10; i < 14; i++) { batch = (await sequentialStore.GetBatch(i + startOffset, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(0, batch.Count); } }
public async Task GetBatchTest() { // Arrange ISequentialStore <Item> sequentialStore = await this.GetSequentialStore("invalidGetBatch"); // Try to get the batch, should return empty batch. List <(long, Item)> batch = (await sequentialStore.GetBatch(0, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(0, batch.Count); // Add 10 elements and remove 4, so that the range of elements is 4 - 9 for (int i = 0; i < 10; i++) { long offset = await sequentialStore.Append(new Item { Prop1 = i }); Assert.Equal(i, offset); } for (int i = 0; i < 4; i++) { await sequentialStore.RemoveFirst((o, itm) => Task.FromResult(true)); } // Try to get with starting offset <= 4, should return remaining elements - 4 - 9. for (int i = 0; i <= 4; i++) { batch = (await sequentialStore.GetBatch(i, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(10 - 4, batch.Count); Assert.Equal(4, batch[0].Item1); } // Try to get with starting offset between 5 and 9, should return a valid batch. for (int i = 5; i < 10; i++) { batch = (await sequentialStore.GetBatch(i, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(10 - i, batch.Count); Assert.Equal(i, batch[0].Item1); } // Try to get with starting offset > 10, should return empty batch for (int i = 10; i < 14; i++) { batch = (await sequentialStore.GetBatch(i, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(0, batch.Count); } }
public async Task BasicTest() { ISequentialStore <Item> sequentialStore = await this.GetSequentialStore("basicTest"); var tasks = new List <Task>(); for (int i = 0; i < 10; i++) { int i1 = i; tasks.Add(Task.Run(async() => { for (int j = 0; j < 1000; j++) { long offset = await sequentialStore.Append(new Item { Prop1 = i1 * 10 + j }); Assert.True(offset <= 10000); } })); } await Task.WhenAll(tasks); IEnumerable <(long offset, Item item)> batch = await sequentialStore.GetBatch(0, 10000); IEnumerable <(long offset, Item item)> batchItems = batch as IList <(long offset, Item item)> ?? batch.ToList(); Assert.Equal(10000, batchItems.Count()); int counter = 0; foreach ((long offset, Item item)batchItem in batchItems) { Assert.Equal(counter++, batchItem.offset); } }
public async Task CreateWithOffsetTest() { IEntityStore <byte[], Item> entityStore = this.GetEntityStore <Item>($"testEntity{Guid.NewGuid().ToString()}"); ISequentialStore <Item> sequentialStore = await SequentialStore <Item> .Create(entityStore, 10150); long offset = await sequentialStore.Append(new Item { Prop1 = 10 }); Assert.Equal(10150, offset); sequentialStore = await SequentialStore <Item> .Create(entityStore); for (int i = 0; i < 10; i++) { offset = await sequentialStore.Append(new Item { Prop1 = 20 }); Assert.Equal(10151 + i, offset); } sequentialStore = await SequentialStore <Item> .Create(entityStore); offset = await sequentialStore.Append(new Item { Prop1 = 20 }); Assert.Equal(10161, offset); IList <(long, Item)> batch = (await sequentialStore.GetBatch(0, 100)).ToList(); Assert.Equal(12, batch.Count); long expectedOffset = 10150; foreach ((long itemOffset, Item _) in batch) { Assert.Equal(itemOffset, expectedOffset++); } }
private async Task ProcessAllAsync <T>(ISequentialStore <T> store, Action <T> callback) { int batchSize = 10; long lastKey = 0; var batch = await store.GetBatch(lastKey, batchSize); while (batch.Any()) { foreach ((long, T)item in batch) { lastKey = item.Item1; callback(item.Item2); } batch = await store.GetBatch(lastKey + 1, batchSize); } }
public async Task BasicTest(Option <long> defaultHeadOffset) { string entityId = $"basicTest{Guid.NewGuid().ToString()}"; ISequentialStore <Item> sequentialStore = await this.GetSequentialStore(entityId, defaultHeadOffset); long startOffset = defaultHeadOffset.GetOrElse(0); var tasks = new List <Task>(); for (int i = 0; i < 10; i++) { int i1 = i; tasks.Add( Task.Run( async() => { for (int j = 0; j < 1000; j++) { long offset = await sequentialStore.Append(new Item { Prop1 = i1 * 10 + j }); Assert.True(offset <= 10000 + startOffset); } })); } await Task.WhenAll(tasks); IEnumerable <(long offset, Item item)> batch = await sequentialStore.GetBatch(startOffset, 10000); IEnumerable <(long offset, Item item)> batchItems = batch as IList <(long offset, Item item)> ?? batch.ToList(); Assert.Equal(10000, batchItems.Count()); long counter = startOffset; foreach ((long offset, Item item)batchItem in batchItems) { Assert.Equal(counter++, batchItem.offset); } }
private async Task CleanQueue(bool checkEntireQueueOnCleanup) { long totalCleanupCount = 0; long totalCleanupStoreCount = 0; while (true) { foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores) { var messageQueueId = endpointSequentialStore.Key; try { if (this.cancellationTokenSource.IsCancellationRequested) { return; } var(endpointId, priority) = MessageQueueIdHelper.ParseMessageQueueId(messageQueueId); Events.CleanupTaskStarted(messageQueueId); CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(messageQueueId, CancellationToken.None); ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value; Events.CleanupCheckpointState(messageQueueId, checkpointData); int cleanupEntityStoreCount = 0; // If checkEntireQueueOnCleanup is set to false, we only peek the head, message counts is tailOffset-headOffset+1 // otherwise count while iterating over the queue. var headOffset = 0L; var tailOffset = sequentialStore.GetTailOffset(CancellationToken.None); var messageCount = 0L; async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef) { var expiry = messageRef.TimeStamp + messageRef.TimeToLive; if (offset > checkpointData.Offset && expiry > DateTime.UtcNow) { // message is not sent and not expired, increase message counts messageCount++; return(false); } headOffset = Math.Max(headOffset, offset); bool deleteMessage = false; // Decrement ref count. var message = await this.messageStore.messageEntityStore.Update( messageRef.EdgeMessageId, m => { if (m.RefCount > 0) { m.RefCount--; } if (m.RefCount == 0) { deleteMessage = true; } return(m); }); if (deleteMessage) { if (offset > checkpointData.Offset && expiry <= DateTime.UtcNow) { this.expiredCounter.Increment(1, new[] { "ttl_expiry", message?.Message.GetSenderId(), message?.Message.GetOutput(), bool.TrueString }); } await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId); cleanupEntityStoreCount++; } return(true); } // With the addition of PriorityQueues, the CleanupProcessor assumptions change slightly: // Previously, we could always assume that if a message at the head of the queue should not be deleted, // then none of the other messages in the queue should be either. Now, because we can have different TTL's // for messages within the same queue, there can be messages that have expired in the queue after the head. // The checkEntireQueueOnCleanup flag is an environment variable for edgeHub. If it is set to true, we will // check the entire queue every time cleanup processor runs. If it is set to false, we just remove the oldest // items in the queue until we get to one that is not expired. int cleanupCount = 0; if (checkEntireQueueOnCleanup) { IEnumerable <(long, MessageRef)> batch; long offset = sequentialStore.GetHeadOffset(this.cancellationTokenSource.Token); do { batch = await sequentialStore.GetBatch(offset, CleanupBatchSize); foreach ((long, MessageRef)messageWithOffset in batch) { if (await sequentialStore.RemoveOffset(DeleteMessageCallback, messageWithOffset.Item1, this.cancellationTokenSource.Token)) { cleanupCount++; } } offset += CleanupBatchSize; }while (batch.Any()); } else { while (await sequentialStore.RemoveFirst(DeleteMessageCallback)) { cleanupCount++; } messageCount = tailOffset - headOffset + 1; } // update Metrics for message counts Checkpointer.Metrics.QueueLength.Set(messageCount, new[] { endpointId, priority.ToString(), bool.TrueString }); totalCleanupCount += cleanupCount; totalCleanupStoreCount += cleanupEntityStoreCount; Events.CleanupCompleted(messageQueueId, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount); await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token); } catch (Exception ex) { Events.ErrorCleaningMessagesForEndpoint(ex, messageQueueId); } } await Task.Delay(this.GetCleanupTaskSleepTime()); } }
private async Task CleanQueue(bool checkEntireQueueOnCleanup) { long totalCleanupCount = 0; long totalCleanupStoreCount = 0; while (true) { foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores) { try { if (this.cancellationTokenSource.IsCancellationRequested) { return; } Events.CleanupTaskStarted(endpointSequentialStore.Key); CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(endpointSequentialStore.Key, CancellationToken.None); ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value; Events.CleanupCheckpointState(endpointSequentialStore.Key, checkpointData); int cleanupEntityStoreCount = 0; async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef) { if (checkpointData.Offset < offset && DateTime.UtcNow - messageRef.TimeStamp < messageRef.TimeToLive) { return(false); } bool deleteMessage = false; // Decrement ref count. await this.messageStore.messageEntityStore.Update( messageRef.EdgeMessageId, m => { if (m.RefCount > 0) { m.RefCount--; } if (m.RefCount == 0) { deleteMessage = true; } return(m); }); if (deleteMessage) { await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId); cleanupEntityStoreCount++; } return(true); } // With the addition of PriorityQueues, the CleanupProcessor assumptions change slightly: // Previously, we could always assume that if a message at the head of the queue should not be deleted, // then none of the other messages in the queue should be either. Now, because we can have different TTL's // for messages within the same queue, there can be messages that have expired in the queue after the head. // The checkEntireQueueOnCleanup flag is an environment variable for edgeHub. If it is set to true, we will // check the entire queue every time cleanup processor runs. If it is set to false, we just remove the oldest // items in the queue until we get to one that is not expired. int cleanupCount = 0; if (checkEntireQueueOnCleanup) { IEnumerable <(long, MessageRef)> batch; long offset = sequentialStore.GetHeadOffset(this.cancellationTokenSource.Token); do { batch = await sequentialStore.GetBatch(offset, CleanupBatchSize); foreach ((long, MessageRef)messageWithOffset in batch) { if (await sequentialStore.RemoveOffset(DeleteMessageCallback, messageWithOffset.Item1, this.cancellationTokenSource.Token)) { cleanupCount++; } } offset = offset + CleanupBatchSize; }while (batch.Any()); } else { while (await sequentialStore.RemoveFirst(DeleteMessageCallback)) { cleanupCount++; } } totalCleanupCount += cleanupCount; totalCleanupStoreCount += cleanupEntityStoreCount; Events.CleanupCompleted(endpointSequentialStore.Key, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount); await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token); } catch (Exception ex) { Events.ErrorCleaningMessagesForEndpoint(ex, endpointSequentialStore.Key); } } await Task.Delay(this.GetCleanupTaskSleepTime()); } }
public async Task RemoveTest(Option <long> defaultHeadOffset) { string entityId = $"removeTestEntity{Guid.NewGuid().ToString()}"; long startOffset = defaultHeadOffset.GetOrElse(0); ISequentialStore <Item> sequentialStore = await this.GetSequentialStore(entityId, defaultHeadOffset); for (int i = 0; i < 10; i++) { long offset = await sequentialStore.Append(new Item { Prop1 = i }); Assert.Equal(i + startOffset, offset); } List <(long offset, Item item)> batch = (await sequentialStore.GetBatch(startOffset, 100)).ToList(); Assert.Equal(10, batch.Count); long counter = startOffset; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 0)); batch = (await sequentialStore.GetBatch(startOffset, 100)).ToList(); Assert.Equal(9, batch.Count); counter = startOffset + 1; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } batch = (await sequentialStore.GetBatch(startOffset + 1, 100)).ToList(); Assert.Equal(9, batch.Count); counter = startOffset + 1; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 0)); batch = (await sequentialStore.GetBatch(startOffset + 1, 100)).ToList(); Assert.Equal(9, batch.Count); counter = startOffset + 1; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 1)); batch = (await sequentialStore.GetBatch(startOffset + 2, 100)).ToList(); Assert.Equal(8, batch.Count); counter = startOffset + 2; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } }