private unsafe void AppendNewCompressedBlock(byte[] compressedBytes, int compressedBytesCount, int originalBytesCount)
{
var offsetForCurrentCompressedBlock = _header.NextCompressedBlockOffseet;
_indexAndHeaderStorage.SetPositionOffset(sizeof(CompressedStorageHeader) + _header.CompressedBlocksCount * 8);
_indexAndHeaderStorage.WriteStructure(ref offsetForCurrentCompressedBlock);
++_header.CompressedBlocksCount;
_header.NextCompressedBlockOffseet += compressedBytesCount + 8; // COMPRESSED BYTES + COUNT(4) + COUNT ORIGINAL(4)
_header.TotalCompressedBytesCount += compressedBytesCount;
_header.TotalProcessedBytesCount += originalBytesCount;
if (compressedBytesCount > _header.MaxCompressedBufferSize)
{
_header.MaxCompressedBufferSize = compressedBytesCount;
}
_indexAndHeaderStorage.SetPositionOffset(0);
_indexAndHeaderStorage.WriteStructure(ref _header);
_blockOffsets.Add(offsetForCurrentCompressedBlock);
_compressedStorage.SetPositionOffset(offsetForCurrentCompressedBlock);
ulong blockHeader = (uint)compressedBytesCount | ((ulong)originalBytesCount << 31);
_compressedStorage.WriteStructure(ref blockHeader);
_compressedStorage.Write(compressedBytes.AsSpan().Slice(0, compressedBytesCount));
_metricsSource.SetMetricValue(MetricsConstants.COMPRESSED_BLOCKS_COUNT_I32, _header.CompressedBlocksCount);
_metricsSource.SetMetricValue(MetricsConstants.TOTAL_COMPRESSED_BYTES_COUNT_I64, _header.TotalCompressedBytesCount);
_metricsSource.SetMetricValue(MetricsConstants.TOTAL_PROCESSED_BYTES_COUNT_I64, _header.TotalProcessedBytesCount);
}
public override void Create()
{
_indexAndHeaderStorage.Create();
_appendOnlyDataStorage.Create();
_compressedStorage.Create();
_header = new CompressedStorageHeader
{
CompressedBlocksCount = 0,
NextCompressedBlockOffseet = 0,
TotalCompressedBytesCount = 0,
TotalProcessedBytesCount = 0,
RemainingBytesInUncompressedStorage = 0
};
_indexAndHeaderStorage.WriteStructure(0, ref _header);
Initialize();
}
private unsafe void CreateNewTable()
{
// Initialize table.
_tableHeaderStorage.Create();
_logsStorage.Create();
_logsDataStorage.Create();
_categoriesStorage.Create();
var datetime = DateTime.UtcNow;
var now = datetime.ToTimestamp();
_tableHeaderStructure = new RaxTableHeader
{
MinTimestamp = now,
MaxTimestamp = now,
TotalLogsCount = 0,
CategoriesCount = 0,
LogsDataNextFreeOffset = 0
};
GetOrCreateYearIndexStorage(datetime.Year);
_tableHeaderStorage.WriteStructure(ref _tableHeaderStructure);
CreateInitialCategories();
void CreateInitialCategories()
{
var categoryBuffer = ArrayPool <byte> .Shared.Rent(_catSize);
WriteCategoryToStorage(categoryBuffer, CreateCategory("TRACE"));
WriteCategoryToStorage(categoryBuffer, CreateCategory("DEBUG"));
WriteCategoryToStorage(categoryBuffer, CreateCategory("INFO"));
WriteCategoryToStorage(categoryBuffer, CreateCategory("WARNING"));
WriteCategoryToStorage(categoryBuffer, CreateCategory("ERROR"));
WriteCategoryToStorage(categoryBuffer, CreateCategory("CRITICAL"));
ArrayPool <byte> .Shared.Return(categoryBuffer);
}
AppendLogs(new RaxNewLog[]
{
new RaxNewLog
{
Message = "SYSTEM READY.".ToCharArray(),
Categories = new[] { "@SYS_INFO" },
Timestamp = now
}
});
}