public async Task InitAsync() { const int snapshotId = 0; var fullSnapshot = await InitQueueSnapshot(); _queueSnapshotCache.Init(fullSnapshot, snapshotId); _queueSnapshotWriter.Init(snapshotId); var tasks = new List <Task>(); foreach (var topicAndQueuesSnapshot in fullSnapshot) { var pageId = MessagesContentPagesUtils.GetPageId(topicAndQueuesSnapshot.MessageId); var task = _schedulerByTopic.ExecuteTaskAsync(topicAndQueuesSnapshot.TopicId, pageId, "Init", async() => { await _restorePageFromBlobOperation.TryRestoreFromUncompressedPage(topicAndQueuesSnapshot.TopicId, pageId); }); tasks.Add(task); } await Task.WhenAll(tasks); _appGlobalFlags.Initialized = true; _appLogger.AddLog(LogProcess.System, null, "SYSTEM", "Application Initialized"); }
public static IReadOnlyList <MessagePageId> GetActivePages(this TopicAndQueuesSnapshotGrpcModel snapshot) { var result = new List <MessagePageId>(); var maxPageId = MessagesContentPagesUtils.GetPageId(snapshot.MessageId); result.PopulatePages(maxPageId, maxPageId); foreach (var queueSnapshot in snapshot.QueueSnapshots) { if (queueSnapshot.Ranges != null) { foreach (var range in queueSnapshot.Ranges) { var pageId = MessagesContentPagesUtils.GetPageId(range.FromId); result.PopulatePages(pageId, maxPageId); pageId = MessagesContentPagesUtils.GetPageId(range.ToId); result.PopulatePages(pageId, maxPageId); } } } return(result); }
public bool PageCanBeCompressed(string topicId, MessagePageId pageId) { var snapshot = _queueSnapshotCache.Get(); var topicSnapshot = snapshot.Cache.FirstOrDefault(itm => itm.TopicId == topicId); if (topicSnapshot == null) { return(false); } var activePageId = MessagesContentPagesUtils.GetPageId(topicSnapshot.MessageId); return(pageId.Value < activePageId.Value - 1); }
public async Task <MessageContentModel> GetMessageFromLegacyAsync([FromQuery] string topicId, [FromQuery] long messageId) { var pageId = MessagesContentPagesUtils.GetPageId(messageId); var compressedPage = await ServiceLocator.LegacyCompressedMessagesStorage.GetCompressedPageAsync(topicId, pageId); if (compressedPage.ZippedContent.IsEmpty) { return new MessageContentModel { Id = -1, } } ; var page = compressedPage.ToReadOnlyContentPage(); var message = page.TryGet(messageId); foreach (var grpcModel in page.GetMessages()) { var messagePageId = MessagesContentPagesUtils.GetPageId(grpcModel.MessageId); if (messagePageId.Value != pageId.Value) { Console.WriteLine(grpcModel.MessageId + " does not belong to the page: " + pageId.Value); } } return(new MessageContentModel { Id = message.MessageId, Content = message.Data == null ? null : Convert.ToBase64String(message.Data), Created = message.Created, PageSize = page.Count }); }
public async ValueTask SaveMessagesAsync(IAsyncEnumerable <CompressedMessageChunkModel> request) { if (!ServiceLocator.AppGlobalFlags.Initialized) { throw new Exception("App is not initialized yet"); } if (ServiceLocator.AppGlobalFlags.IsShuttingDown) { throw new Exception("App is stopping"); } var contract = await request.DecompressAndMerge <SaveMessagesGrpcContract>(); if (contract.Messages == null) { Console.WriteLine(contract.TopicId + ": Request to Save messages with empty content"); return; } ServiceLocator.IndexByMinuteWriter.NewMessages(contract.TopicId, contract.Messages); var groups = contract.Messages .GroupBy(itm => MessagesContentPagesUtils.GetPageId(itm.MessageId).Value); foreach (var group in groups) { var messagePageId = new MessagePageId(group.Key); var writablePage = ServiceLocator.MessagesContentCache.TryGetWritablePage(contract.TopicId, messagePageId); if (writablePage != null) { writablePage.NewMessages(group); } else { var result = ServiceLocator.MessagesContentCache.CreateWritablePage(contract.TopicId, messagePageId); if (result.Result != null) { result.Result.NewMessages(group); continue; } if (result.Exists != null) { if (result.Exists is WritableContentPage writableContentPage) { ServiceLocator.AppLogger.AddLog(LogProcess.NewMessages, contract.TopicId, "PageId: " + group.Key, "Creating new writable content page which exists. Reusing it"); writableContentPage.NewMessages(group); continue; } ServiceLocator.AppLogger.AddLog(LogProcess.NewMessages, contract.TopicId, "PageId: " + group.Key, "Trying to add messages, but readOnly messages content is found. Converting it into Writable"); ServiceLocator.MessagesContentCache .ConvertIntoWritable(contract.TopicId, result.Exists).NewMessages(group); continue; } ServiceLocator.AppLogger.AddLog(LogProcess.NewMessages, contract.TopicId, "PageId: " + group.Key, "Trying to add messages, but readOnly messages content is found. Should not be here. Skipping messages"); } } }
public static MessagePageId CreateFromMessageId(long messageId) { return(MessagesContentPagesUtils.GetPageId(messageId)); }
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IHostApplicationLifetime applicationLifetime) { applicationLifetime.ApplicationStopping.Register(() => { ServiceLocator.StopAsync().Wait(); foreach (var(topic, metrics) in ServiceLocator.MetricsByTopic.Get()) { ServiceLocator.AppLogger.AddLog(LogProcess.System, topic, "Last Save message before shutdown", metrics.MaxSavedMessageId.ToString()); var pageId = MessagesContentPagesUtils.GetPageId(metrics.MaxSavedMessageId); var page = ServiceLocator.MessagesContentCache.TryGetPage(topic, pageId); if (page != null) { var messages = page.GetMessages(); if (messages.Count > 0) { var minId = messages.Min(itm => itm.MessageId); var maxId = messages.Max(itm => itm.MessageId); ServiceLocator.AppLogger.AddLog(LogProcess.System, topic, "Last messages in Cache before shutdown", $"Min={minId};Max={maxId}"); } } } var snapshot = ((AppLogger)ServiceLocator.AppLogger).GetSnapshot(); ServiceLocator.LogsSnapshotRepository.SaveAsync(snapshot).AsTask().Wait(); }); if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); } app.UseStaticFiles(); app.UseOpenApi(); app.UseSwaggerUi3(); app.UseGlobalFlagsHandler(); app.UseRouting(); app.UseEndpoints(endpoints => { endpoints.MapControllers(); endpoints.MapGrpcService <MyServiceBusMessagesPersistenceGrpcService>(); endpoints.MapGrpcService <MyServiceBusQueuePersistenceGrpcService>(); endpoints.MapGrpcService <MyServiceBusHistoryReaderGrpcService>(); endpoints.MapMetrics(); }); var sp = _services.BuildServiceProvider(); ServiceLocator.Init(sp, Settings); ServiceLocator.Start(); }