public async Task InitAsync()
{
const int snapshotId = 0;
var fullSnapshot = await InitQueueSnapshot();
_queueSnapshotCache.Init(fullSnapshot, snapshotId);
_queueSnapshotWriter.Init(snapshotId);
var tasks = new List <Task>();
foreach (var topicAndQueuesSnapshot in fullSnapshot)
{
var pageId = MessagesContentPagesUtils.GetPageId(topicAndQueuesSnapshot.MessageId);
var task = _schedulerByTopic.ExecuteTaskAsync(topicAndQueuesSnapshot.TopicId, pageId, "Init", async() =>
{
await _restorePageFromBlobOperation.TryRestoreFromUncompressedPage(topicAndQueuesSnapshot.TopicId, pageId);
});
tasks.Add(task);
}
await Task.WhenAll(tasks);
_appGlobalFlags.Initialized = true;
_appLogger.AddLog(LogProcess.System, null, "SYSTEM", "Application Initialized");
}
public static IReadOnlyList <MessagePageId> GetActivePages(this TopicAndQueuesSnapshotGrpcModel snapshot)
{
var result = new List <MessagePageId>();
var maxPageId = MessagesContentPagesUtils.GetPageId(snapshot.MessageId);
result.PopulatePages(maxPageId, maxPageId);
foreach (var queueSnapshot in snapshot.QueueSnapshots)
{
if (queueSnapshot.Ranges != null)
{
foreach (var range in queueSnapshot.Ranges)
{
var pageId = MessagesContentPagesUtils.GetPageId(range.FromId);
result.PopulatePages(pageId, maxPageId);
pageId = MessagesContentPagesUtils.GetPageId(range.ToId);
result.PopulatePages(pageId, maxPageId);
}
}
}
return(result);
}
public bool PageCanBeCompressed(string topicId, MessagePageId pageId)
{
var snapshot = _queueSnapshotCache.Get();
var topicSnapshot = snapshot.Cache.FirstOrDefault(itm => itm.TopicId == topicId);
if (topicSnapshot == null)
{
return(false);
}
var activePageId = MessagesContentPagesUtils.GetPageId(topicSnapshot.MessageId);
return(pageId.Value < activePageId.Value - 1);
}
public async Task <MessageContentModel> GetMessageFromLegacyAsync([FromQuery] string topicId, [FromQuery] long messageId)
{
var pageId = MessagesContentPagesUtils.GetPageId(messageId);
var compressedPage = await ServiceLocator.LegacyCompressedMessagesStorage.GetCompressedPageAsync(topicId, pageId);
if (compressedPage.ZippedContent.IsEmpty)
{
return new MessageContentModel
{
Id = -1,
}
}
;
var page = compressedPage.ToReadOnlyContentPage();
var message = page.TryGet(messageId);
foreach (var grpcModel in page.GetMessages())
{
var messagePageId = MessagesContentPagesUtils.GetPageId(grpcModel.MessageId);
if (messagePageId.Value != pageId.Value)
{
Console.WriteLine(grpcModel.MessageId + " does not belong to the page: " + pageId.Value);
}
}
return(new MessageContentModel
{
Id = message.MessageId,
Content = message.Data == null ? null : Convert.ToBase64String(message.Data),
Created = message.Created,
PageSize = page.Count
});
}
public async ValueTask SaveMessagesAsync(IAsyncEnumerable <CompressedMessageChunkModel> request)
{
if (!ServiceLocator.AppGlobalFlags.Initialized)
{
throw new Exception("App is not initialized yet");
}
if (ServiceLocator.AppGlobalFlags.IsShuttingDown)
{
throw new Exception("App is stopping");
}
var contract = await request.DecompressAndMerge <SaveMessagesGrpcContract>();
if (contract.Messages == null)
{
Console.WriteLine(contract.TopicId + ": Request to Save messages with empty content");
return;
}
ServiceLocator.IndexByMinuteWriter.NewMessages(contract.TopicId, contract.Messages);
var groups = contract.Messages
.GroupBy(itm => MessagesContentPagesUtils.GetPageId(itm.MessageId).Value);
foreach (var group in groups)
{
var messagePageId = new MessagePageId(group.Key);
var writablePage = ServiceLocator.MessagesContentCache.TryGetWritablePage(contract.TopicId, messagePageId);
if (writablePage != null)
{
writablePage.NewMessages(group);
}
else
{
var result =
ServiceLocator.MessagesContentCache.CreateWritablePage(contract.TopicId, messagePageId);
if (result.Result != null)
{
result.Result.NewMessages(group);
continue;
}
if (result.Exists != null)
{
if (result.Exists is WritableContentPage writableContentPage)
{
ServiceLocator.AppLogger.AddLog(LogProcess.NewMessages, contract.TopicId,
"PageId: " + group.Key, "Creating new writable content page which exists. Reusing it");
writableContentPage.NewMessages(group);
continue;
}
ServiceLocator.AppLogger.AddLog(LogProcess.NewMessages, contract.TopicId,
"PageId: " + group.Key,
"Trying to add messages, but readOnly messages content is found. Converting it into Writable");
ServiceLocator.MessagesContentCache
.ConvertIntoWritable(contract.TopicId, result.Exists).NewMessages(group);
continue;
}
ServiceLocator.AppLogger.AddLog(LogProcess.NewMessages, contract.TopicId,
"PageId: " + group.Key,
"Trying to add messages, but readOnly messages content is found. Should not be here. Skipping messages");
}
}
}
public static MessagePageId CreateFromMessageId(long messageId)
{
return(MessagesContentPagesUtils.GetPageId(messageId));
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IHostApplicationLifetime applicationLifetime)
{
applicationLifetime.ApplicationStopping.Register(() =>
{
ServiceLocator.StopAsync().Wait();
foreach (var(topic, metrics) in ServiceLocator.MetricsByTopic.Get())
{
ServiceLocator.AppLogger.AddLog(LogProcess.System, topic, "Last Save message before shutdown", metrics.MaxSavedMessageId.ToString());
var pageId = MessagesContentPagesUtils.GetPageId(metrics.MaxSavedMessageId);
var page = ServiceLocator.MessagesContentCache.TryGetPage(topic, pageId);
if (page != null)
{
var messages = page.GetMessages();
if (messages.Count > 0)
{
var minId = messages.Min(itm => itm.MessageId);
var maxId = messages.Max(itm => itm.MessageId);
ServiceLocator.AppLogger.AddLog(LogProcess.System, topic, "Last messages in Cache before shutdown", $"Min={minId};Max={maxId}");
}
}
}
var snapshot = ((AppLogger)ServiceLocator.AppLogger).GetSnapshot();
ServiceLocator.LogsSnapshotRepository.SaveAsync(snapshot).AsTask().Wait();
});
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
app.UseStaticFiles();
app.UseOpenApi();
app.UseSwaggerUi3();
app.UseGlobalFlagsHandler();
app.UseRouting();
app.UseEndpoints(endpoints =>
{
endpoints.MapControllers();
endpoints.MapGrpcService <MyServiceBusMessagesPersistenceGrpcService>();
endpoints.MapGrpcService <MyServiceBusQueuePersistenceGrpcService>();
endpoints.MapGrpcService <MyServiceBusHistoryReaderGrpcService>();
endpoints.MapMetrics();
});
var sp = _services.BuildServiceProvider();
ServiceLocator.Init(sp, Settings);
ServiceLocator.Start();
}