Beispiel #1
0
        private unsafe void AppendNewCompressedBlock(byte[] compressedBytes, int compressedBytesCount, int originalBytesCount)
        {
            var offsetForCurrentCompressedBlock = _header.NextCompressedBlockOffseet;

            _indexAndHeaderStorage.SetPositionOffset(sizeof(CompressedStorageHeader) + _header.CompressedBlocksCount * 8);
            _indexAndHeaderStorage.WriteStructure(ref offsetForCurrentCompressedBlock);

            ++_header.CompressedBlocksCount;
            _header.NextCompressedBlockOffseet += compressedBytesCount + 8;             // COMPRESSED BYTES + COUNT(4) + COUNT ORIGINAL(4)
            _header.TotalCompressedBytesCount  += compressedBytesCount;
            _header.TotalProcessedBytesCount   += originalBytesCount;
            if (compressedBytesCount > _header.MaxCompressedBufferSize)
            {
                _header.MaxCompressedBufferSize = compressedBytesCount;
            }

            _indexAndHeaderStorage.SetPositionOffset(0);
            _indexAndHeaderStorage.WriteStructure(ref _header);

            _blockOffsets.Add(offsetForCurrentCompressedBlock);

            _compressedStorage.SetPositionOffset(offsetForCurrentCompressedBlock);
            ulong blockHeader = (uint)compressedBytesCount | ((ulong)originalBytesCount << 31);

            _compressedStorage.WriteStructure(ref blockHeader);
            _compressedStorage.Write(compressedBytes.AsSpan().Slice(0, compressedBytesCount));

            _metricsSource.SetMetricValue(MetricsConstants.COMPRESSED_BLOCKS_COUNT_I32, _header.CompressedBlocksCount);
            _metricsSource.SetMetricValue(MetricsConstants.TOTAL_COMPRESSED_BYTES_COUNT_I64, _header.TotalCompressedBytesCount);
            _metricsSource.SetMetricValue(MetricsConstants.TOTAL_PROCESSED_BYTES_COUNT_I64, _header.TotalProcessedBytesCount);
        }
Beispiel #2
0
        public override void Open()
        {
            _indexAndHeaderStorage.Open();
            _appendOnlyDataStorage.Open();
            _compressedStorage.Open();

            _indexAndHeaderStorage.SetPositionOffset(0);
            _indexAndHeaderStorage.ReadStructure(ref _header);

            Initialize();

            if (_header.CompressedBlocksCount > 0)
            {
                var offsetsBufferBytesCount = _header.CompressedBlocksCount * 8;
                var offsetsBuffer           = new byte[offsetsBufferBytesCount];
                _indexAndHeaderStorage.ReadStructureArray(offsetsBuffer);

                var offsets = MemoryMarshal.Cast <byte, long>(offsetsBuffer);
                for (int i = 0; i < _header.CompressedBlocksCount; i++)
                {
                    _blockOffsets.Add(offsets[i]);
                }
            }
        }
Beispiel #3
0
        public override void Read(Span <byte> buffer)
        {
            if (_header.CompressedBlocksCount == 0)
            {
                _appendOnlyDataStorage.SetPositionOffset(_currentAbsoluteOffset);
                _appendOnlyDataStorage.Read(buffer);
                return;
            }

            var absoluteOffset = _currentAbsoluteOffset;
            var blockIndex     = 0;

            var     reusableBufferSpan       = _reusableBuffer.AsSpan();
            var     compressedBuffer         = new byte[_header.MaxCompressedBufferSize + 8];
            var     fullCompressedBufferSpan = compressedBuffer.AsSpan();
            ref var headerOfCompressedBuffer = ref MemoryMarshal.Cast <byte, ulong>(fullCompressedBufferSpan.Slice(0, 8))[0];
Beispiel #4
0
        private void OpenExistingTable()
        {
            // Attempt to read table data.
            _tableHeaderStorage.Open();
            _logsStorage.Open();
            _logsDataStorage.Open();
            _categoriesStorage.Open();

            _tableHeaderStorage.ReadStructure(ref _tableHeaderStructure);

            // Load categories.
            var oneCategoryEntry = default(RaxCategoryEntry);

            for (int i = 0; i < _tableHeaderStructure.CategoriesCount; i++)
            {
                var category = new CategoryInformation();

                _categoriesStorage.SetPositionOffset(i * _catSize);
                _categoriesStorage.ReadStructure(ref oneCategoryEntry);

                category.CategoryIndex = i;
                category.CategoryName  = new string(oneCategoryEntry.CategoryName, 0, oneCategoryEntry.CategoryRealNameLengthInBytes / 2);
                category.LogsCount     = oneCategoryEntry.Header.LogsCountInCategory;
                category.MinTimestamp  = oneCategoryEntry.Header.MinTimestamp;
                category.MaxTimestamp  = oneCategoryEntry.Header.MaxTimestamp;

                _categoriesByIndex[category.CategoryIndex] = category;
                _categoriesByName[category.CategoryName]   = category;
            }

            // Load indexes
            for (int year = new DateTime(_tableHeaderStructure.MinTimestamp).Year;
                 year <= new DateTime(_tableHeaderStructure.MaxTimestamp).Year;
                 ++year)
            {
                GetOrCreateYearIndexStorage(year);
            }
        }
Beispiel #5
0
        public RaxLogEntry[] GetLogs(long minTimestamp, long maxTimestamp, long skipLogs = 0, long takeLogs = long.MaxValue)
        {
            var range = GetLogsRange(minTimestamp, maxTimestamp, skipLogs, takeLogs);

            var result = new RaxLogEntry[range.LogsCount];

            if (range.LogsCount == 0)
            {
                return(result);
            }

            // MAX_SELECT_LOGS_BUFFER
            var bufferSize = range.LogsCount * _logSize;

            var buffer = new byte[bufferSize];

            _logsStorage.SetPositionOffset(range.FromLogIndex * _logSize);
            _logsStorage.ReadStructureArray(buffer);

            var logs = MemoryMarshal.Cast <byte, RaxFixedSizeLogEntry>(buffer);

            var fromLog = logs[0];
            var toLog   = logs[logs.Length - 1];

            var fromOffset    = fromLog.DataOffset;
            var toOffset      = toLog.DataOffset;
            var logsBytesData = toOffset + toLog.TextLengthInBytes + toLog.CategoriesCount * 8;

            // MAX_SELECT_LOGS_BUFFER
            var dataBuffer = new byte[logsBytesData];

            _logsDataStorage.SetPositionOffset(fromOffset);
            _logsDataStorage.ReadStructureArray(dataBuffer);

            var dataBufferSpan = dataBuffer.AsSpan();

            var dataOffset = 0;

            for (int i = 0; i < logs.Length; i++)
            {
                var log = logs[i];

                var dataSpanBegin = dataBufferSpan.Slice(dataOffset);
                var text          = new string(MemoryMarshal.Cast <byte, char>(dataSpanBegin.Slice(0, log.TextLengthInBytes)));
                var datetime      = new DateTime(log.Timestamp);

                var categories = new string[log.CategoriesCount];

                var logCategoriesSpan = MemoryMarshal.Cast <byte, long>(dataSpanBegin.Slice(log.TextLengthInBytes, log.CategoriesCount * 8));
                for (int catIndex = 0; catIndex < log.CategoriesCount; catIndex++)
                {
                    categories[catIndex] = _categoriesByIndex[logCategoriesSpan[catIndex]].CategoryName;
                }

                result[i] = new RaxLogEntry
                {
                    Text       = text,
                    Date       = datetime,
                    Categories = categories
                };

                dataOffset += ComputeLogDataSizeInBytes(text.Length, log.CategoriesCount);
            }

            return(result);
        }