private unsafe void AppendNewCompressedBlock(byte[] compressedBytes, int compressedBytesCount, int originalBytesCount)
{
var offsetForCurrentCompressedBlock = _header.NextCompressedBlockOffseet;
_indexAndHeaderStorage.SetPositionOffset(sizeof(CompressedStorageHeader) + _header.CompressedBlocksCount * 8);
_indexAndHeaderStorage.WriteStructure(ref offsetForCurrentCompressedBlock);
++_header.CompressedBlocksCount;
_header.NextCompressedBlockOffseet += compressedBytesCount + 8; // COMPRESSED BYTES + COUNT(4) + COUNT ORIGINAL(4)
_header.TotalCompressedBytesCount += compressedBytesCount;
_header.TotalProcessedBytesCount += originalBytesCount;
if (compressedBytesCount > _header.MaxCompressedBufferSize)
{
_header.MaxCompressedBufferSize = compressedBytesCount;
}
_indexAndHeaderStorage.SetPositionOffset(0);
_indexAndHeaderStorage.WriteStructure(ref _header);
_blockOffsets.Add(offsetForCurrentCompressedBlock);
_compressedStorage.SetPositionOffset(offsetForCurrentCompressedBlock);
ulong blockHeader = (uint)compressedBytesCount | ((ulong)originalBytesCount << 31);
_compressedStorage.WriteStructure(ref blockHeader);
_compressedStorage.Write(compressedBytes.AsSpan().Slice(0, compressedBytesCount));
_metricsSource.SetMetricValue(MetricsConstants.COMPRESSED_BLOCKS_COUNT_I32, _header.CompressedBlocksCount);
_metricsSource.SetMetricValue(MetricsConstants.TOTAL_COMPRESSED_BYTES_COUNT_I64, _header.TotalCompressedBytesCount);
_metricsSource.SetMetricValue(MetricsConstants.TOTAL_PROCESSED_BYTES_COUNT_I64, _header.TotalProcessedBytesCount);
}
public override void Open()
{
_indexAndHeaderStorage.Open();
_appendOnlyDataStorage.Open();
_compressedStorage.Open();
_indexAndHeaderStorage.SetPositionOffset(0);
_indexAndHeaderStorage.ReadStructure(ref _header);
Initialize();
if (_header.CompressedBlocksCount > 0)
{
var offsetsBufferBytesCount = _header.CompressedBlocksCount * 8;
var offsetsBuffer = new byte[offsetsBufferBytesCount];
_indexAndHeaderStorage.ReadStructureArray(offsetsBuffer);
var offsets = MemoryMarshal.Cast <byte, long>(offsetsBuffer);
for (int i = 0; i < _header.CompressedBlocksCount; i++)
{
_blockOffsets.Add(offsets[i]);
}
}
}
public override void Read(Span <byte> buffer)
{
if (_header.CompressedBlocksCount == 0)
{
_appendOnlyDataStorage.SetPositionOffset(_currentAbsoluteOffset);
_appendOnlyDataStorage.Read(buffer);
return;
}
var absoluteOffset = _currentAbsoluteOffset;
var blockIndex = 0;
var reusableBufferSpan = _reusableBuffer.AsSpan();
var compressedBuffer = new byte[_header.MaxCompressedBufferSize + 8];
var fullCompressedBufferSpan = compressedBuffer.AsSpan();
ref var headerOfCompressedBuffer = ref MemoryMarshal.Cast <byte, ulong>(fullCompressedBufferSpan.Slice(0, 8))[0];
private void OpenExistingTable()
{
// Attempt to read table data.
_tableHeaderStorage.Open();
_logsStorage.Open();
_logsDataStorage.Open();
_categoriesStorage.Open();
_tableHeaderStorage.ReadStructure(ref _tableHeaderStructure);
// Load categories.
var oneCategoryEntry = default(RaxCategoryEntry);
for (int i = 0; i < _tableHeaderStructure.CategoriesCount; i++)
{
var category = new CategoryInformation();
_categoriesStorage.SetPositionOffset(i * _catSize);
_categoriesStorage.ReadStructure(ref oneCategoryEntry);
category.CategoryIndex = i;
category.CategoryName = new string(oneCategoryEntry.CategoryName, 0, oneCategoryEntry.CategoryRealNameLengthInBytes / 2);
category.LogsCount = oneCategoryEntry.Header.LogsCountInCategory;
category.MinTimestamp = oneCategoryEntry.Header.MinTimestamp;
category.MaxTimestamp = oneCategoryEntry.Header.MaxTimestamp;
_categoriesByIndex[category.CategoryIndex] = category;
_categoriesByName[category.CategoryName] = category;
}
// Load indexes
for (int year = new DateTime(_tableHeaderStructure.MinTimestamp).Year;
year <= new DateTime(_tableHeaderStructure.MaxTimestamp).Year;
++year)
{
GetOrCreateYearIndexStorage(year);
}
}
public RaxLogEntry[] GetLogs(long minTimestamp, long maxTimestamp, long skipLogs = 0, long takeLogs = long.MaxValue)
{
var range = GetLogsRange(minTimestamp, maxTimestamp, skipLogs, takeLogs);
var result = new RaxLogEntry[range.LogsCount];
if (range.LogsCount == 0)
{
return(result);
}
// MAX_SELECT_LOGS_BUFFER
var bufferSize = range.LogsCount * _logSize;
var buffer = new byte[bufferSize];
_logsStorage.SetPositionOffset(range.FromLogIndex * _logSize);
_logsStorage.ReadStructureArray(buffer);
var logs = MemoryMarshal.Cast <byte, RaxFixedSizeLogEntry>(buffer);
var fromLog = logs[0];
var toLog = logs[logs.Length - 1];
var fromOffset = fromLog.DataOffset;
var toOffset = toLog.DataOffset;
var logsBytesData = toOffset + toLog.TextLengthInBytes + toLog.CategoriesCount * 8;
// MAX_SELECT_LOGS_BUFFER
var dataBuffer = new byte[logsBytesData];
_logsDataStorage.SetPositionOffset(fromOffset);
_logsDataStorage.ReadStructureArray(dataBuffer);
var dataBufferSpan = dataBuffer.AsSpan();
var dataOffset = 0;
for (int i = 0; i < logs.Length; i++)
{
var log = logs[i];
var dataSpanBegin = dataBufferSpan.Slice(dataOffset);
var text = new string(MemoryMarshal.Cast <byte, char>(dataSpanBegin.Slice(0, log.TextLengthInBytes)));
var datetime = new DateTime(log.Timestamp);
var categories = new string[log.CategoriesCount];
var logCategoriesSpan = MemoryMarshal.Cast <byte, long>(dataSpanBegin.Slice(log.TextLengthInBytes, log.CategoriesCount * 8));
for (int catIndex = 0; catIndex < log.CategoriesCount; catIndex++)
{
categories[catIndex] = _categoriesByIndex[logCategoriesSpan[catIndex]].CategoryName;
}
result[i] = new RaxLogEntry
{
Text = text,
Date = datetime,
Categories = categories
};
dataOffset += ComputeLogDataSizeInBytes(text.Length, log.CategoriesCount);
}
return(result);
}