public CompressedPage(MessagePageId pageId, IReadOnlyList <MessageContentGrpcModel> messages)
{
ZippedContent = messages.ZipMessages();
Messages = messages;
PageId = pageId;
CalcMinMax();
}
public CompressedPage(MessagePageId pageId, ReadOnlyMemory <byte> zippedContent)
{
ZippedContent = zippedContent;
Messages = zippedContent.UnzipMessages();
PageId = pageId;
CalcMinMax();
}
private async Task TestPageWriteAndRead(PagesCluster clusterPage, long pageId, Dictionary <long, byte[]> cache)
{
var messagePageId = new MessagePageId(pageId);
var hasPage = await clusterPage.HasPageAsync(messagePageId);
Assert.IsFalse(hasPage);
var msg = new MessageContentGrpcModel
{
Created = DateTime.UtcNow,
Data = new [] { (byte)pageId, (byte)pageId, (byte)pageId },
MessageId = pageId,
};
var content = new CompressedPage(new MessagePageId(pageId), new [] { msg });
cache.Add(pageId, content.ZippedContent.ToArray());
await clusterPage.WriteAsync(messagePageId, content);
hasPage = await clusterPage.HasPageAsync(messagePageId);
Assert.IsTrue(hasPage);
var resultContent = await clusterPage.ReadAsync(messagePageId);
resultContent.ZippedContent.AssertAllBytesAreEqualWith(content.ZippedContent);
}
private async Task DoubleCheck(PagesCluster clusterPage, long pageId, byte[] srcContent)
{
var messagePageId = new MessagePageId(pageId);
var resultContent = await clusterPage.ReadAsync(messagePageId);
Assert.AreEqual(resultContent.Messages[0].MessageId, pageId);
resultContent.ZippedContent.AssertAllBytesAreEqualWith(srcContent);
}
public async ValueTask <bool> HasPageAsync(MessagePageId pageId)
{
var tocIndex = GetPageTocIndex(pageId);
var(_, length) = await GetPagePositionAllocationToc(tocIndex, false);
return(length > 0);
}
public void Update(MessagePageId pageId, long blobPosition, long maxSavedMessageId)
{
if (MaxSavedMessageId < maxSavedMessageId)
{
MaxSavedMessageId = maxSavedMessageId;
}
if (PageId.Value <= pageId.Value)
{
BlobPosition = blobPosition;
PageId = pageId;
}
}
public async Task <CompressPageResult> CompressAsync([FromForm] string topicId, [FromForm] long pageId)
{
var snapshot = ServiceLocator.QueueSnapshotCache.Get();
var topicSnapshot = snapshot.Cache.FirstOrDefault(itm => itm.TopicId == topicId);
if (topicSnapshot == null)
{
return new CompressPageResult
{
Result = "Topic not found"
}
}
;
var messagePageId = new MessagePageId(pageId);
if (!ServiceLocator.CompressedMessagesUtils.PageCanBeCompressed(topicId, messagePageId))
{
return(new CompressPageResult
{
Result = "Can not compress current page"
});
}
await ServiceLocator.TaskSchedulerByTopic.ExecuteTaskAsync(topicId, messagePageId,
"Compressing page by REST API request",
async() =>
{
var page = ServiceLocator.MessagesContentCache.TryGetPage(topicId, messagePageId);
if (page == null)
{
await ServiceLocator.MessagesContentReader.TryGetPageTopicThreadSynchronizedAsync(topicId,
messagePageId);
}
if (page != null)
{
await ServiceLocator.CompressPageBlobOperation.ExecuteOperationThreadTopicSynchronizedAsync(
topicId, messagePageId, page);
}
});
return(new CompressPageResult
{
Result = "Page is compressed"
});
}
public bool PageCanBeCompressed(string topicId, MessagePageId pageId)
{
var snapshot = _queueSnapshotCache.Get();
var topicSnapshot = snapshot.Cache.FirstOrDefault(itm => itm.TopicId == topicId);
if (topicSnapshot == null)
{
return(false);
}
var activePageId = MessagesContentPagesUtils.GetPageId(topicSnapshot.MessageId);
return(pageId.Value < activePageId.Value - 1);
}
public ValueTask SaveLastCompressedPageStorageAsync(string topicId, MessagePageId pageId)
{
lock (_lastPages)
{
if (_lastPages.ContainsKey(topicId))
{
_lastPages[topicId] = pageId.Value;
}
else
{
_lastPages.Add(topicId, pageId.Value);
}
_hasDataToUpdate = true;
}
return(new ValueTask());
}
public async Task WriteAsync(MessagePageId pageId, CompressedPage pageData)
{
var tocIndex = GetPageTocIndex(pageId);
if (_tocPage == null)
{
await InitIndexPageAsync(true);
}
var nextPageNoToWrite = await GetNextPageNoToWriteAsync(tocIndex, pageData.ZippedContent.Length);
await AzurePageBlob.WriteBytesAsync(pageData.ZippedContent, nextPageNoToWrite, new WriteBytesOptions
{
SplitRoundTripsPageSize = 4096
});
await WritePagePositionAllocationToc(tocIndex, nextPageNoToWrite, pageData.ZippedContent.Length);
}
public async ValueTask <IActionResult> Download([Required][FromQuery] string topicId, [Required][FromQuery] long pageId)
{
var messagePageId = new MessagePageId(pageId);
var page = ServiceLocator.MessagesContentCache.TryGetPage(topicId, messagePageId);
if (page == null)
{
await ServiceLocator.TaskSchedulerByTopic.ExecuteTaskAsync(topicId, messagePageId, "Load Debug Page",
async() =>
{
await ServiceLocator.MessagesContentReader.LoadPageIntoCacheTopicSynchronizedAsync(topicId,
messagePageId);
});
}
page = ServiceLocator.MessagesContentCache.TryGetPage(topicId, messagePageId);
return(File(page.GetCompressedPage().ZippedContent.ToArray(), pageId + "application/zip", topicId + '-' + pageId + ".zip"));
}
public async Task <CompressedPage> ReadAsync(MessagePageId pageId)
{
try
{
//If we do not have exceptions - can remove this try/catch
GetPageTocIndex(pageId);
}
catch (Exception e)
{
Console.WriteLine("Can not get TocIndex for page: " + pageId.Value + "; TopicId:" + TopicId);
Console.WriteLine(e);
throw;
}
var tocIndex = GetPageTocIndex(pageId);
var(position, length) = await GetPagePositionAllocationToc(tocIndex, false);
if (length == 0)
{
return(CompressedPage.CreateEmpty(pageId));
}
try
{
//If we do not have exceptions - can remove this try/catch
var fullPages = MyAzurePageBlobUtils.CalculateRequiredPagesAmount(length);
var result = await AzurePageBlob.ReadAsync(position, fullPages);
var buffer = result.GetBuffer();
return(new CompressedPage(pageId, new ReadOnlyMemory <byte>(buffer, 0, (int)length)));
}
catch (Exception)
{
Console.WriteLine("Problem with reading page: " + pageId + " TocIndex: " + tocIndex + "; position:" + position + " Length:" + length);
return(CompressedPage.CreateEmpty(pageId));
}
}
private static void PopulatePages(this List <MessagePageId> pages, MessagePageId pageId, MessagePageId maxPageId)
{
if (pageId.Value <= maxPageId.Value)
{
if (pages.All(itm => itm.Value != pageId.Value))
{
pages.Add(pageId);
}
}
var nextPageId = pageId.Value + 1;
if (nextPageId <= maxPageId.Value)
{
if (pages.All(itm => itm.Value != nextPageId))
{
pages.Add(new MessagePageId(nextPageId));
}
}
}
private async Task GcThreadTopicSynchronizedAsync(string topicId, MessagePageId pageId)
{
var page = _messagesContentCache.TryGetPage(topicId, pageId);
if (page == null)
{
return;
}
var gcResult = await _messagesContentPersistentStorage.TryToGcAsync(topicId, pageId);
if (gcResult.NotFound)
{
_logger.AddLog(LogProcess.PagesLoaderOrGc, topicId, "PageNo:" + pageId, "Attempt to GC PageWriter which is not found. Disposing page from the Cache");
_messagesContentCache.DisposePage(topicId, pageId);
return;
}
if (gcResult.NotReadyToGc)
{
_logger.AddLog(LogProcess.PagesLoaderOrGc, topicId, "PageNo:" + pageId, "Attempt to GC PageWriter which has not synced messages. Trying to Sync them to the Blob and skipping for the next round");
await _messagesContentPersistentStorage.SyncAsync(topicId, pageId);
return;
}
if (gcResult.DisposedPageWriter != null)
{
_logger.AddLog(LogProcess.PagesLoaderOrGc, topicId, "PageNo:" + pageId, "PageWriter disposed Ok.. Disposing From Cache");
await _compressPageBlobOperation.ExecuteOperationThreadTopicSynchronizedAsync(topicId, pageId, gcResult.DisposedPageWriter.AssignedPage);
_messagesContentCache.DisposePage(topicId, pageId);
_logger.AddLog(LogProcess.PagesLoaderOrGc, topicId, "PageNo:" + pageId, "Page is disposed from Cache");
}
}
public Dictionary <string, ActivePagesByTopic> GetActivePages()
{
var(_, cache) = _queueSnapshotCache.Get();
var result = new Dictionary <string, ActivePagesByTopic>();
foreach (var topicSnapshot in cache)
{
var pages = new List <MessagePageId>
{
MessagePageId.CreateFromMessageId(topicSnapshot.MessageId)
};
foreach (var queueSnapshot in topicSnapshot.QueueSnapshots)
{
foreach (var range in queueSnapshot.Ranges)
{
var fromPageId = MessagePageId.CreateFromMessageId(range.FromId);
var toPageId = MessagePageId.CreateFromMessageId(range.ToId);
if (pages.All(itm => itm.Value != fromPageId.Value))
{
pages.Add(fromPageId);
}
if (pages.All(itm => itm.Value != toPageId.Value))
{
pages.Add(toPageId);
}
}
}
result.Add(topicSnapshot.TopicId, new ActivePagesByTopic
{
Snapshot = topicSnapshot,
Pages = pages
});
}
return(result);
}
public async ValueTask <IActionResult> Index([Required][FromQuery] string topicId, [Required][FromQuery] long pageId)
{
var messagePageId = new MessagePageId(pageId);
var page = ServiceLocator.MessagesContentCache.TryGetPage(topicId, messagePageId);
if (page == null)
{
await ServiceLocator.TaskSchedulerByTopic.ExecuteTaskAsync(topicId, messagePageId, "Load Debug Page",
async() =>
{
await ServiceLocator.MessagesContentReader.LoadPageIntoCacheTopicSynchronizedAsync(topicId,
messagePageId);
});
}
page = ServiceLocator.MessagesContentCache.TryGetPage(topicId, messagePageId);
if (page == null)
{
return(NotFound("Page Not Found"));
}
var result = new
{
page.MinMessageId,
page.MaxMessageId,
page.Count,
shouldHaveAmount = page.ShouldHaveAmount(),
hasSkipped = page.HasSkipped(),
notSavedAmount = page.NotSavedAmount,
hashCode = page.GetHashCode()
};
return(Json(result));
}
public static CompressedPage CreateEmpty(MessagePageId pageId)
{
return(new (pageId, ReadOnlyMemory <byte> .Empty));
}
private int GetPageTocIndex(MessagePageId pageId)
{
var firstPageNoOnCompressedPage = ClusterPageId.GetFirstPageIdOnCompressedPage();
return((int)(pageId.Value - firstPageNoOnCompressedPage.Value));
}
public async ValueTask SaveMessagesAsync(IAsyncEnumerable <CompressedMessageChunkModel> request)
{
if (!ServiceLocator.AppGlobalFlags.Initialized)
{
throw new Exception("App is not initialized yet");
}
if (ServiceLocator.AppGlobalFlags.IsShuttingDown)
{
throw new Exception("App is stopping");
}
var contract = await request.DecompressAndMerge <SaveMessagesGrpcContract>();
if (contract.Messages == null)
{
Console.WriteLine(contract.TopicId + ": Request to Save messages with empty content");
return;
}
ServiceLocator.IndexByMinuteWriter.NewMessages(contract.TopicId, contract.Messages);
var groups = contract.Messages
.GroupBy(itm => MessagesContentPagesUtils.GetPageId(itm.MessageId).Value);
foreach (var group in groups)
{
var messagePageId = new MessagePageId(group.Key);
var writablePage = ServiceLocator.MessagesContentCache.TryGetWritablePage(contract.TopicId, messagePageId);
if (writablePage != null)
{
writablePage.NewMessages(group);
}
else
{
var result =
ServiceLocator.MessagesContentCache.CreateWritablePage(contract.TopicId, messagePageId);
if (result.Result != null)
{
result.Result.NewMessages(group);
continue;
}
if (result.Exists != null)
{
if (result.Exists is WritableContentPage writableContentPage)
{
ServiceLocator.AppLogger.AddLog(LogProcess.NewMessages, contract.TopicId,
"PageId: " + group.Key, "Creating new writable content page which exists. Reusing it");
writableContentPage.NewMessages(group);
continue;
}
ServiceLocator.AppLogger.AddLog(LogProcess.NewMessages, contract.TopicId,
"PageId: " + group.Key,
"Trying to add messages, but readOnly messages content is found. Converting it into Writable");
ServiceLocator.MessagesContentCache
.ConvertIntoWritable(contract.TopicId, result.Exists).NewMessages(group);
continue;
}
ServiceLocator.AppLogger.AddLog(LogProcess.NewMessages, contract.TopicId,
"PageId: " + group.Key,
"Trying to add messages, but readOnly messages content is found. Should not be here. Skipping messages");
}
}
}
public async Task ExecuteOperationThreadTopicSynchronizedAsync(string topicId, MessagePageId pageId, IMessageContentPage messageContentPage)
{
var logContext = "Page: " + pageId.Value;
if (await _compressedMessagesStorage.HasCompressedPageAsync(topicId, pageId))
{
_appLogger.AddLog(LogProcess.PagesCompressor, topicId, logContext, "Has compressed page. Skipping compressing procedure");
return;
}
if (messageContentPage.Count == 0)
{
_appLogger.AddLog(LogProcess.PagesCompressor, topicId, logContext, "No messages to compress. Skipping compressing procedure");
return;
}
var compressedPage = messageContentPage.GetCompressedPage();
if (_appGlobalFlags.DebugTopic == topicId)
{
_appLogger.AddLog(LogProcess.PagesCompressor, topicId, logContext, $"Writing Compressed data for page {pageId}.");
}
await _compressedMessagesStorage.WriteCompressedPageAsync(topicId, pageId, compressedPage, _appLogger);
if (_appGlobalFlags.DebugTopic == topicId)
{
_appLogger.AddLog(LogProcess.PagesCompressor, topicId, logContext, $"Verifying compressed data for page {pageId}");
}
var compressedPageToVerify = await _compressedMessagesStorage.GetCompressedPageAsync(topicId, pageId);
var messages = compressedPageToVerify.Messages;
if (_appGlobalFlags.DebugTopic == topicId)
{
_appLogger.AddLog(LogProcess.PagesCompressor, topicId, logContext, $"Verified compressed data for page {pageId}. Messages: " + messages.Count);
}
if (_appGlobalFlags.DebugTopic == topicId)
{
_appLogger.AddLog(LogProcess.PagesCompressor, topicId, logContext, $"Deleting Uncompressed page {pageId}");
}
await _persistentStorage.DeleteNonCompressedPageAsync(topicId, pageId);
if (_appGlobalFlags.DebugTopic == topicId)
{
_appLogger.AddLog(LogProcess.PagesCompressor, topicId, logContext, "Written Compressed Page: " + pageId + ". Messages in the page:" + messageContentPage.Count);
}
}
public static MessagePageId PrevPage(this MessagePageId pageId)
{
return(new MessagePageId(pageId.Value + 1));
}
public bool EqualsWith(MessagePageId messagePageId)
{
return(Value == messagePageId.Value);
}
private async Task WarmUpThreadTopicSynchronizedAsync(string topicId, MessagePageId pageId)
{
_logger.AddLog(LogProcess.PagesLoaderOrGc, topicId, "PageNo:" + pageId, "Warming up the page");
await _messagesContentReader.LoadPageIntoCacheTopicSynchronizedAsync(topicId, pageId);
}
public ReadOnlyContentPage(CompressedPage compressedPage)
{
PageId = compressedPage.PageId;
_compressedPage = compressedPage;
}
public async Task <IMessageContentPage> TryRestoreFromCompressedPage(string topicId, MessagePageId pageId)
{
var resultFromCache = _messagesContentCache.TryGetPage(topicId, pageId);
if (resultFromCache != null)
{
return(resultFromCache);
}
var logContext = "PageId: " + pageId.Value;
_appLogger.AddLog(LogProcess.PagesLoaderOrGc, topicId, logContext,
$"Restoring page #{pageId} from compressed source");
var pageCompressedContent = await _compressedMessagesStorage.GetCompressedPageAsync(topicId, pageId);
var dt = DateTime.UtcNow;
if (pageCompressedContent.ZippedContent.Length == 0)
{
_appLogger.AddLog(LogProcess.PagesLoaderOrGc, topicId, logContext,
$"Can not restore page #{pageId} from compressed source. Duration: {DateTime.UtcNow - dt}");
return(null);
}
var msgs = pageCompressedContent.Messages;
long minId = 0;
long maxId = 0;
if (msgs.Count > 0)
{
minId = msgs.Min(itm => itm.MessageId);
maxId = msgs.Max(itm => itm.MessageId);
}
_appLogger.AddLog(LogProcess.PagesLoaderOrGc, topicId, logContext,
$"Restored page #{pageId} from compressed source. Duration: {DateTime.UtcNow - dt}. Messages: {msgs.Count}. MinId: {minId}, MaxId: {maxId}");
var result = new ReadOnlyContentPage(pageCompressedContent);
_messagesContentCache.AddPage(topicId, result);
return(result);
}
public static ClusterPageId GetClusterPageId(this MessagePageId messagePageId)
{
var result = messagePageId.Value / PagesOnCluster;
return(new ClusterPageId(result));
}
public MessagesPage(MessagePageId pageId)
{
PageId = pageId;
}
public async Task <IMessageContentPage> TryRestoreFromUncompressedPage(string topicId, MessagePageId pageId)
{
var logContext = "PageId: " + pageId.Value;
var resultFromCache = _messagesContentCache.TryGetPage(topicId, pageId);
if (resultFromCache != null)
{
return(resultFromCache);
}
_appLogger.AddLog(LogProcess.PagesLoaderOrGc, topicId, logContext,
$"Restoring page #{pageId} from UnCompressed source");
var dt = DateTime.UtcNow;
var pageWriter = await _messagesContentPersistentStorage.TryGetAsync(topicId, pageId,
() =>
{
var result = _messagesContentCache.CreateWritablePage(topicId, pageId);
if (result.Result != null)
{
return(result.Result);
}
if (result.Exists != null)
{
if (result.Exists is WritableContentPage existingWritablePage)
{
_appLogger.AddLog(LogProcess.PagesLoaderOrGc, topicId, logContext, "Restoring uncompressed page and found existing writable content page. Using it");
return(existingWritablePage);
}
throw new Exception(
$"Trying to create page {topicId}/{pageId} by found non writable content page {result.Exists.GetType()}");
}
throw new Exception($"RestorePageFromBlobOperation.TryRestoreFromUncompressedPage I should not be here. {topicId}/{pageId}");
});
if (pageWriter == null)
{
_appLogger.AddLog(LogProcess.PagesLoaderOrGc, topicId, logContext,
"Can not restore page from uncompressed page");
return(null);
}
_appLogger.AddLog(LogProcess.PagesLoaderOrGc, topicId, logContext,
$"Restored page #{pageId} from UnCompressed source. Duration: {DateTime.UtcNow - dt}. Messages: " +
pageWriter.AssignedPage.Count);
return(pageWriter.AssignedPage);
}