public async Task BasicTest() { ISequentialStore <Item> sequentialStore = await this.GetSequentialStore("basicTest"); var tasks = new List <Task>(); for (int i = 0; i < 10; i++) { int i1 = i; tasks.Add(Task.Run(async() => { for (int j = 0; j < 1000; j++) { long offset = await sequentialStore.Append(new Item { Prop1 = i1 * 10 + j }); Assert.True(offset <= 10000); } })); } await Task.WhenAll(tasks); IEnumerable <(long offset, Item item)> batch = await sequentialStore.GetBatch(0, 10000); IEnumerable <(long offset, Item item)> batchItems = batch as IList <(long offset, Item item)> ?? batch.ToList(); Assert.Equal(10000, batchItems.Count()); int counter = 0; foreach ((long offset, Item item)batchItem in batchItems) { Assert.Equal(counter++, batchItem.offset); } }
public StoreTestResultCollection(ISequentialStore <T> store, int batchSize) { this.batchSize = batchSize; this.store = store; this.resultQueue = new Queue <T>(); this.lastLoadedPosition = -1; }
public async Task <ISequentialStore <T> > GetSequentialStore <T>(string entityName) { IEntityStore <byte[], T> underlyingStore = this.GetEntityStore <byte[], T>(entityName); ISequentialStore <T> sequentialStore = await SequentialStore <T> .Create(underlyingStore); return(sequentialStore); }
async Task <ISequentialStore <Item> > GetSequentialStore(string entityName) { IEntityStore <byte[], Item> entityStore = this.GetEntityStore <Item>(entityName); ISequentialStore <Item> sequentialStore = await SequentialStore <Item> .Create(entityStore); return(sequentialStore); }
public async Task CreateWithOffsetTest() { IEntityStore <byte[], Item> entityStore = this.GetEntityStore <Item>($"testEntity{Guid.NewGuid().ToString()}"); ISequentialStore <Item> sequentialStore = await SequentialStore <Item> .Create(entityStore, 10150); long offset = await sequentialStore.Append(new Item { Prop1 = 10 }); Assert.Equal(10150, offset); sequentialStore = await SequentialStore <Item> .Create(entityStore); for (int i = 0; i < 10; i++) { offset = await sequentialStore.Append(new Item { Prop1 = 20 }); Assert.Equal(10151 + i, offset); } sequentialStore = await SequentialStore <Item> .Create(entityStore); offset = await sequentialStore.Append(new Item { Prop1 = 20 }); Assert.Equal(10161, offset); IList <(long, Item)> batch = (await sequentialStore.GetBatch(0, 100)).ToList(); Assert.Equal(12, batch.Count); long expectedOffset = 10150; foreach ((long itemOffset, Item _) in batch) { Assert.Equal(itemOffset, expectedOffset++); } }
public async Task InitAsync(string storagePath, ISystemEnvironment systemEnvironment, bool optimizeForPerformance) { StoreProvider storeProvider; try { var partitionsList = new List <string> { "messages", "directMethods", "twins" }; IDbStoreProvider dbStoreprovider = DbStoreProvider.Create( new RocksDbOptionsProvider(systemEnvironment, optimizeForPerformance, Option.None <ulong>(), Option.None <int>(), Option.None <StorageLogLevel>()), this.GetStoragePath(storagePath), partitionsList); storeProvider = new StoreProvider(dbStoreprovider); } catch (Exception ex) when(!ExceptionEx.IsFatal(ex)) { Logger.LogError(ex, "Error creating RocksDB store. Falling back to in-memory store."); storeProvider = new StoreProvider(new InMemoryDbStoreProvider()); } this.messagesStore = await storeProvider.GetSequentialStore <MessageDetails>(MessageStorePartitionKey); this.directMethodsStore = await storeProvider.GetSequentialStore <TestOperationResult>(DirectMethodsStorePartitionKey); this.twinsStore = await storeProvider.GetSequentialStore <TestOperationResult>(TwinsStorePartitionKey); }
public async Task RemoveTest() { ISequentialStore <Item> sequentialStore = await this.GetSequentialStore("removeTestEntity"); for (int i = 0; i < 10; i++) { long offset = await sequentialStore.Append(new Item { Prop1 = i }); Assert.Equal(i, offset); } List <(long offset, Item item)> batch = (await sequentialStore.GetBatch(0, 100)).ToList(); Assert.Equal(10, batch.Count); int counter = 0; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 0)); batch = (await sequentialStore.GetBatch(0, 100)).ToList(); Assert.Equal(9, batch.Count); counter = 1; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } batch = (await sequentialStore.GetBatch(1, 100)).ToList(); Assert.Equal(9, batch.Count); counter = 1; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 0)); batch = (await sequentialStore.GetBatch(1, 100)).ToList(); Assert.Equal(9, batch.Count); counter = 1; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 1)); batch = (await sequentialStore.GetBatch(2, 100)).ToList(); Assert.Equal(8, batch.Count); counter = 2; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } }
public MessageIterator(IKeyValueStore <string, MessageWrapper> entityStore, ISequentialStore <MessageRef> endpointSequentialStore, long startingOffset) { this.entityStore = entityStore; this.endpointSequentialStore = endpointSequentialStore; this.startingOffset = startingOffset < 0 ? 0 : startingOffset; }
internal StoreTestResultCollectionEnumerator(ISequentialStore <T> store, int batchSize) { this.batchSize = batchSize; this.store = store; this.resultQueue = new Queue <T>(); this.lastLoadedPosition = -1; this.current = default(T); }
public async Task AddEndpoint(string endpointId) { ISequentialStore <MessageRef> sequentialStore = await this.storeProvider.GetSequentialStore <MessageRef>(endpointId); if (this.endpointSequentialStores.TryAdd(endpointId, sequentialStore)) { Events.SequentialStoreAdded(endpointId); } }
async Task <ISequentialStore <Item> > GetSequentialStore(string entityName, Option <long> defaultHeadOffset) { IEntityStore <byte[], Item> entityStore = this.GetEntityStore <Item>(entityName); ISequentialStore <Item> sequentialStore = await defaultHeadOffset .Map(d => SequentialStore <Item> .Create(entityStore, d)) .GetOrElse(() => SequentialStore <Item> .Create(entityStore)); return(sequentialStore); }
public async Task AddEndpoint(string endpointId) { CheckpointData checkpointData = await this.checkpointStore.GetCheckpointDataAsync(endpointId, CancellationToken.None); ISequentialStore <MessageRef> sequentialStore = await this.storeProvider.GetSequentialStore <MessageRef>(endpointId, checkpointData.Offset + 1); if (this.endpointSequentialStores.TryAdd(endpointId, sequentialStore)) { Events.SequentialStoreAdded(endpointId); } }
public async Task GetBatchTest(Option <long> defaultHeadOffset) { // Arrange string entityId = $"invalidGetBatch{Guid.NewGuid().ToString()}"; long startOffset = defaultHeadOffset.GetOrElse(0); ISequentialStore <Item> sequentialStore = await this.GetSequentialStore(entityId, defaultHeadOffset); // Try to get the batch, should return empty batch. List <(long, Item)> batch = (await sequentialStore.GetBatch(startOffset, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(0, batch.Count); // Add 10 elements and remove 4, so that the range of elements is 4 - 9 for (int i = 0; i < 10; i++) { long offset = await sequentialStore.Append(new Item { Prop1 = i }); Assert.Equal(i + startOffset, offset); } for (int i = 0; i < 4; i++) { await sequentialStore.RemoveFirst((o, itm) => Task.FromResult(true)); } // Try to get with starting offset <= 4, should return remaining elements - 4 - 9. for (int i = 0; i <= 4; i++) { batch = (await sequentialStore.GetBatch(i + startOffset, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(10 - 4, batch.Count); Assert.Equal(4 + startOffset, batch[0].Item1); } // Try to get with starting offset between 5 and 9, should return a valid batch. for (int i = 5; i < 10; i++) { batch = (await sequentialStore.GetBatch(i + startOffset, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(10 - i, batch.Count); Assert.Equal(i + startOffset, batch[0].Item1); } // Try to get with starting offset > 10, should return empty batch for (int i = 10; i < 14; i++) { batch = (await sequentialStore.GetBatch(i + startOffset, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(0, batch.Count); } }
public async Task GetBatchTest() { // Arrange ISequentialStore <Item> sequentialStore = await this.GetSequentialStore("invalidGetBatch"); // Try to get the batch, should return empty batch. List <(long, Item)> batch = (await sequentialStore.GetBatch(0, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(0, batch.Count); // Add 10 elements and remove 4, so that the range of elements is 4 - 9 for (int i = 0; i < 10; i++) { long offset = await sequentialStore.Append(new Item { Prop1 = i }); Assert.Equal(i, offset); } for (int i = 0; i < 4; i++) { await sequentialStore.RemoveFirst((o, itm) => Task.FromResult(true)); } // Try to get with starting offset <= 4, should return remaining elements - 4 - 9. for (int i = 0; i <= 4; i++) { batch = (await sequentialStore.GetBatch(i, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(10 - 4, batch.Count); Assert.Equal(4, batch[0].Item1); } // Try to get with starting offset between 5 and 9, should return a valid batch. for (int i = 5; i < 10; i++) { batch = (await sequentialStore.GetBatch(i, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(10 - i, batch.Count); Assert.Equal(i, batch[0].Item1); } // Try to get with starting offset > 10, should return empty batch for (int i = 10; i < 14; i++) { batch = (await sequentialStore.GetBatch(i, 10)).ToList(); Assert.NotNull(batch); Assert.Equal(0, batch.Count); } }
public async Task CreateTest() { IEntityStore <byte[], Item> entityStore = this.GetEntityStore <Item>("testEntity"); ISequentialStore <Item> sequentialStore = await SequentialStore <Item> .Create(entityStore); long offset = await sequentialStore.Append(new Item { Prop1 = 10 }); Assert.Equal(0, offset); sequentialStore = await SequentialStore <Item> .Create(entityStore); offset = await sequentialStore.Append(new Item { Prop1 = 20 }); Assert.Equal(1, offset); }
public async Task CreateTest() { string entityName = $"testEntity{Guid.NewGuid().ToString()}"; IEntityStore <byte[], Item> entityStore = this.GetEntityStore <Item>(entityName); ISequentialStore <Item> sequentialStore = await SequentialStore <Item> .Create(entityStore, entityName); long offset = await sequentialStore.Append(new Item { Prop1 = 10 }); Assert.Equal(0, offset); sequentialStore = await SequentialStore <Item> .Create(entityStore, entityName); offset = await sequentialStore.Append(new Item { Prop1 = 20 }); Assert.Equal(1, offset); }
private async Task ProcessAllAsync <T>(ISequentialStore <T> store, Action <T> callback) { int batchSize = 10; long lastKey = 0; var batch = await store.GetBatch(lastKey, batchSize); while (batch.Any()) { foreach ((long, T)item in batch) { lastKey = item.Item1; callback(item.Item2); } batch = await store.GetBatch(lastKey + 1, batchSize); } }
public async Task BasicTest(Option <long> defaultHeadOffset) { string entityId = $"basicTest{Guid.NewGuid().ToString()}"; ISequentialStore <Item> sequentialStore = await this.GetSequentialStore(entityId, defaultHeadOffset); long startOffset = defaultHeadOffset.GetOrElse(0); var tasks = new List <Task>(); for (int i = 0; i < 10; i++) { int i1 = i; tasks.Add( Task.Run( async() => { for (int j = 0; j < 1000; j++) { long offset = await sequentialStore.Append(new Item { Prop1 = i1 * 10 + j }); Assert.True(offset <= 10000 + startOffset); } })); } await Task.WhenAll(tasks); IEnumerable <(long offset, Item item)> batch = await sequentialStore.GetBatch(startOffset, 10000); IEnumerable <(long offset, Item item)> batchItems = batch as IList <(long offset, Item item)> ?? batch.ToList(); Assert.Equal(10000, batchItems.Count()); long counter = startOffset; foreach ((long offset, Item item)batchItem in batchItems) { Assert.Equal(counter++, batchItem.offset); } }
protected override void Load(ContainerBuilder builder) { // IMetricsScraper builder.Register(c => new MetricsScraper(new string[] { "http://edgeHub:9600/metrics", "http://edgeAgent:9600/metrics" })) .As <IMetricsScraper>() .SingleInstance(); // Task<IMetricsStorage> builder.Register(async c => { IStoreProvider storeProvider = await c.Resolve <Task <IStoreProvider> >(); ISequentialStore <IEnumerable <Metric> > dataStore = await storeProvider.GetSequentialStore <IEnumerable <Metric> >("Metrics"); return(new MetricsStorage(dataStore) as IMetricsStorage); }) .As <Task <IMetricsStorage> >() .SingleInstance(); // IMetricsPublisher builder.RegisterType <EdgeRuntimeDiagnosticsUpload>() .As <IMetricsPublisher>() .SingleInstance(); // Task<MetricsWorker> builder.Register(async c => { IMetricsScraper scraper = c.Resolve <IMetricsScraper>(); IMetricsPublisher publisher = c.Resolve <IMetricsPublisher>(); Task <IMetricsStorage> storage = c.Resolve <Task <IMetricsStorage> >(); return(new MetricsWorker(scraper, await storage, publisher)); }) .As <Task <MetricsWorker> >() .SingleInstance(); base.Load(builder); }
/// <summary> /// Messages need to be cleaned up in the following scenarios – /// 1.When the message expires (exceeds TTL) /// 2.When a message has been processed (indicated by the the checkpoint) /// 3.When there are 0 references to a message in the store from the message queues, /// the message itself is deleted from the store (this means the message was successfully delivered to all endpoints). /// // TODO - Update cleanup logic to cleanup expired messages from entity store as well. /// </summary> async Task CleanupMessages() { long totalCleanupCount = 0; long totalCleanupStoreCount = 0; try { while (true) { foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores) { try { if (this.cancellationTokenSource.IsCancellationRequested) { return; } Events.CleanupTaskStarted(endpointSequentialStore.Key); CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(endpointSequentialStore.Key, CancellationToken.None); ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value; Events.CleanupCheckpointState(endpointSequentialStore.Key, checkpointData); int cleanupEntityStoreCount = 0; async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef) { if (checkpointData.Offset < offset && DateTime.UtcNow - messageRef.TimeStamp < messageRef.TimeToLive) { return(false); } bool deleteMessage = false; // Decrement ref count. await this.messageStore.messageEntityStore.Update( messageRef.EdgeMessageId, m => { if (m.RefCount > 0) { m.RefCount--; } if (m.RefCount == 0) { deleteMessage = true; } return(m); }); if (deleteMessage) { await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId); cleanupEntityStoreCount++; } return(true); } // With the addition of PriorityQueues, the CleanupProcessor assumptions change slightly: // Previously, we could always assume that if a message at the head of the queue should not be deleted, // then none of the other messages in the queue should be either. Now, because we can have different TTL's // for messages within the same queue, there can be messages that have expired in the queue after the head. // This is okay because they will be cleaned up eventually and they will be ignored otherwise. // TODO: Add optional CleanupProcessor mode that will go through the entire length of the queue each time to remove expired messages. int cleanupCount = 0; while (await sequentialStore.RemoveFirst(DeleteMessageCallback)) { cleanupCount++; } totalCleanupCount += cleanupCount; totalCleanupStoreCount += cleanupEntityStoreCount; Events.CleanupCompleted(endpointSequentialStore.Key, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount); await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token); } catch (Exception ex) { Events.ErrorCleaningMessagesForEndpoint(ex, endpointSequentialStore.Key); } } await Task.Delay(this.GetCleanupTaskSleepTime()); } } catch (Exception ex) { Events.ErrorCleaningMessages(ex); throw; } }
private async Task CleanQueue(bool checkEntireQueueOnCleanup) { long totalCleanupCount = 0; long totalCleanupStoreCount = 0; while (true) { foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores) { var messageQueueId = endpointSequentialStore.Key; try { if (this.cancellationTokenSource.IsCancellationRequested) { return; } var(endpointId, priority) = MessageQueueIdHelper.ParseMessageQueueId(messageQueueId); Events.CleanupTaskStarted(messageQueueId); CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(messageQueueId, CancellationToken.None); ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value; Events.CleanupCheckpointState(messageQueueId, checkpointData); int cleanupEntityStoreCount = 0; // If checkEntireQueueOnCleanup is set to false, we only peek the head, message counts is tailOffset-headOffset+1 // otherwise count while iterating over the queue. var headOffset = 0L; var tailOffset = sequentialStore.GetTailOffset(CancellationToken.None); var messageCount = 0L; async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef) { var expiry = messageRef.TimeStamp + messageRef.TimeToLive; if (offset > checkpointData.Offset && expiry > DateTime.UtcNow) { // message is not sent and not expired, increase message counts messageCount++; return(false); } headOffset = Math.Max(headOffset, offset); bool deleteMessage = false; // Decrement ref count. var message = await this.messageStore.messageEntityStore.Update( messageRef.EdgeMessageId, m => { if (m.RefCount > 0) { m.RefCount--; } if (m.RefCount == 0) { deleteMessage = true; } return(m); }); if (deleteMessage) { if (offset > checkpointData.Offset && expiry <= DateTime.UtcNow) { this.expiredCounter.Increment(1, new[] { "ttl_expiry", message?.Message.GetSenderId(), message?.Message.GetOutput(), bool.TrueString }); } await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId); cleanupEntityStoreCount++; } return(true); } // With the addition of PriorityQueues, the CleanupProcessor assumptions change slightly: // Previously, we could always assume that if a message at the head of the queue should not be deleted, // then none of the other messages in the queue should be either. Now, because we can have different TTL's // for messages within the same queue, there can be messages that have expired in the queue after the head. // The checkEntireQueueOnCleanup flag is an environment variable for edgeHub. If it is set to true, we will // check the entire queue every time cleanup processor runs. If it is set to false, we just remove the oldest // items in the queue until we get to one that is not expired. int cleanupCount = 0; if (checkEntireQueueOnCleanup) { IEnumerable <(long, MessageRef)> batch; long offset = sequentialStore.GetHeadOffset(this.cancellationTokenSource.Token); do { batch = await sequentialStore.GetBatch(offset, CleanupBatchSize); foreach ((long, MessageRef)messageWithOffset in batch) { if (await sequentialStore.RemoveOffset(DeleteMessageCallback, messageWithOffset.Item1, this.cancellationTokenSource.Token)) { cleanupCount++; } } offset += CleanupBatchSize; }while (batch.Any()); } else { while (await sequentialStore.RemoveFirst(DeleteMessageCallback)) { cleanupCount++; } messageCount = tailOffset - headOffset + 1; } // update Metrics for message counts Checkpointer.Metrics.QueueLength.Set(messageCount, new[] { endpointId, priority.ToString(), bool.TrueString }); totalCleanupCount += cleanupCount; totalCleanupStoreCount += cleanupEntityStoreCount; Events.CleanupCompleted(messageQueueId, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount); await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token); } catch (Exception ex) { Events.ErrorCleaningMessagesForEndpoint(ex, messageQueueId); } } await Task.Delay(this.GetCleanupTaskSleepTime()); } }
public Task RemoveStore <T>(ISequentialStore <T> sequentialStore) { this.dbStoreProvider.RemoveDbStore(sequentialStore.EntityName); return(Task.CompletedTask); }
private async Task CleanQueue(bool checkEntireQueueOnCleanup) { long totalCleanupCount = 0; long totalCleanupStoreCount = 0; while (true) { foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores) { try { if (this.cancellationTokenSource.IsCancellationRequested) { return; } Events.CleanupTaskStarted(endpointSequentialStore.Key); CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(endpointSequentialStore.Key, CancellationToken.None); ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value; Events.CleanupCheckpointState(endpointSequentialStore.Key, checkpointData); int cleanupEntityStoreCount = 0; async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef) { if (checkpointData.Offset < offset && DateTime.UtcNow - messageRef.TimeStamp < messageRef.TimeToLive) { return(false); } bool deleteMessage = false; // Decrement ref count. await this.messageStore.messageEntityStore.Update( messageRef.EdgeMessageId, m => { if (m.RefCount > 0) { m.RefCount--; } if (m.RefCount == 0) { deleteMessage = true; } return(m); }); if (deleteMessage) { await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId); cleanupEntityStoreCount++; } return(true); } // With the addition of PriorityQueues, the CleanupProcessor assumptions change slightly: // Previously, we could always assume that if a message at the head of the queue should not be deleted, // then none of the other messages in the queue should be either. Now, because we can have different TTL's // for messages within the same queue, there can be messages that have expired in the queue after the head. // The checkEntireQueueOnCleanup flag is an environment variable for edgeHub. If it is set to true, we will // check the entire queue every time cleanup processor runs. If it is set to false, we just remove the oldest // items in the queue until we get to one that is not expired. int cleanupCount = 0; if (checkEntireQueueOnCleanup) { IEnumerable <(long, MessageRef)> batch; long offset = sequentialStore.GetHeadOffset(this.cancellationTokenSource.Token); do { batch = await sequentialStore.GetBatch(offset, CleanupBatchSize); foreach ((long, MessageRef)messageWithOffset in batch) { if (await sequentialStore.RemoveOffset(DeleteMessageCallback, messageWithOffset.Item1, this.cancellationTokenSource.Token)) { cleanupCount++; } } offset = offset + CleanupBatchSize; }while (batch.Any()); } else { while (await sequentialStore.RemoveFirst(DeleteMessageCallback)) { cleanupCount++; } } totalCleanupCount += cleanupCount; totalCleanupStoreCount += cleanupEntityStoreCount; Events.CleanupCompleted(endpointSequentialStore.Key, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount); await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token); } catch (Exception ex) { Events.ErrorCleaningMessagesForEndpoint(ex, endpointSequentialStore.Key); } } await Task.Delay(this.GetCleanupTaskSleepTime()); } }
public StoreTestResultCollection(ISequentialStore <T> store, int batchSize) { this.enumerator = new StoreTestResultCollectionEnumerator(store, batchSize); }
public MetricsStorage(ISequentialStore <IEnumerable <Metric> > sequentialStore, TimeSpan maxMetricAge) { this.dataStore = Preconditions.CheckNotNull(sequentialStore, nameof(sequentialStore)); this.maxMetricAge = Preconditions.CheckNotNull(maxMetricAge, nameof(maxMetricAge)); }
/// <summary> /// Messages need to be cleaned up in the following scenarios – /// 1. When the message expires (exceeds TTL) /// 2. When a message has been processed (indicated by the the checkpoint) /// 3. When there are 0 references to a message in the store from the message queues, /// the message itself is deleted from the store (this means the message was successfully delivered to all endpoints). /// // TODO - Update cleanup logic to cleanup expired messages from entity store as well. /// </summary> async Task CleanupMessages() { long totalCleanupCount = 0; long totalCleanupStoreCount = 0; try { TimeSpan cleanupTaskSleepTime = this.messageStore.timeToLive.TotalSeconds / 2 < CleanupTaskFrequency.TotalSeconds ? TimeSpan.FromSeconds(this.messageStore.timeToLive.TotalSeconds / 2) : CleanupTaskFrequency; while (true) { foreach (KeyValuePair <string, ISequentialStore <MessageRef> > endpointSequentialStore in this.messageStore.endpointSequentialStores) { try { if (this.cancellationTokenSource.IsCancellationRequested) { return; } Events.CleanupTaskStarted(endpointSequentialStore.Key); CheckpointData checkpointData = await this.messageStore.checkpointStore.GetCheckpointDataAsync(endpointSequentialStore.Key, CancellationToken.None); ISequentialStore <MessageRef> sequentialStore = endpointSequentialStore.Value; Events.CleanupCheckpointState(endpointSequentialStore.Key, checkpointData); int cleanupEntityStoreCount = 0; async Task <bool> DeleteMessageCallback(long offset, MessageRef messageRef) { if (checkpointData.Offset < offset && DateTime.UtcNow - messageRef.TimeStamp < this.messageStore.timeToLive) { return(false); } bool deleteMessage = false; // Decrement ref count. await this.messageStore.messageEntityStore.Update( messageRef.EdgeMessageId, m => { if (m.RefCount > 0) { m.RefCount--; } if (m.RefCount == 0) { deleteMessage = true; } return(m); }); if (deleteMessage) { await this.messageStore.messageEntityStore.Remove(messageRef.EdgeMessageId); cleanupEntityStoreCount++; } return(true); } int cleanupCount = 0; while (await sequentialStore.RemoveFirst(DeleteMessageCallback)) { cleanupCount++; } totalCleanupCount += cleanupCount; totalCleanupStoreCount += cleanupEntityStoreCount; Events.CleanupCompleted(endpointSequentialStore.Key, cleanupCount, cleanupEntityStoreCount, totalCleanupCount, totalCleanupStoreCount); await Task.Delay(MinCleanupSleepTime, this.cancellationTokenSource.Token); } catch (Exception ex) { Events.ErrorCleaningMessagesForEndpoint(ex, endpointSequentialStore.Key); } } await Task.Delay(cleanupTaskSleepTime); } } catch (Exception ex) { Events.ErrorCleaningMessages(ex); throw; } }
public async Task RemoveTest(Option <long> defaultHeadOffset) { string entityId = $"removeTestEntity{Guid.NewGuid().ToString()}"; long startOffset = defaultHeadOffset.GetOrElse(0); ISequentialStore <Item> sequentialStore = await this.GetSequentialStore(entityId, defaultHeadOffset); for (int i = 0; i < 10; i++) { long offset = await sequentialStore.Append(new Item { Prop1 = i }); Assert.Equal(i + startOffset, offset); } List <(long offset, Item item)> batch = (await sequentialStore.GetBatch(startOffset, 100)).ToList(); Assert.Equal(10, batch.Count); long counter = startOffset; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 0)); batch = (await sequentialStore.GetBatch(startOffset, 100)).ToList(); Assert.Equal(9, batch.Count); counter = startOffset + 1; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } batch = (await sequentialStore.GetBatch(startOffset + 1, 100)).ToList(); Assert.Equal(9, batch.Count); counter = startOffset + 1; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 0)); batch = (await sequentialStore.GetBatch(startOffset + 1, 100)).ToList(); Assert.Equal(9, batch.Count); counter = startOffset + 1; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } await sequentialStore.RemoveFirst((offset, item) => Task.FromResult(item.Prop1 == 1)); batch = (await sequentialStore.GetBatch(startOffset + 2, 100)).ToList(); Assert.Equal(8, batch.Count); counter = startOffset + 2; foreach ((long offset, Item item)batchItem in batch) { Assert.Equal(counter++, batchItem.offset); } }