public bool Open(IPositionLessStream positionLessStream, bool dispose) { lock (_log) { _log.WriteUInt8((byte)KVReplayOperation.Open); ulong size = positionLessStream.GetSize(); _log.WriteVUInt64(size); ulong pos = 0; var buf = new byte[4096]; while (pos < size) { var read = positionLessStream.Read(buf, 0, buf.Length, pos); // Next 2 conditions should not happen or file is mutated when it should not if (read == 0) { break; } if ((ulong)read > size - pos) { read = (int)(size - pos); } _log.WriteBlock(buf, 0, read); pos += (ulong)read; } while (pos < size) { _log.WriteUInt8(0); pos++; } _log.FlushBuffer(); } return(_db.Open(positionLessStream, dispose)); }
void StoreHashUpdate(ByteBuffer key, StorageValue storageValue) { if (_hashIndexWriter == null) { StartNewHashIndexFile(); } _hashIndexWriter.WriteVUInt32(storageValue.FileId); _hashIndexWriter.WriteVUInt32(storageValue.FileOfs); _hashIndexWriter.WriteVUInt32(storageValue.ContentLengthCompressedIsLeaf); _hashIndexWriter.WriteBlock(key); }
public void Put(ByteBuffer key, ByteBuffer content) { if (key.Length != _keySize) { throw new ArgumentException("Key has wrong Length not equal to KeySize"); } if (content.Length == 0) { throw new ArgumentException("Empty Content cannot be stored"); } var k = new ByteStructs.Key20(key); CacheValue cacheValue; if (_cache.TryGetValue(k, out cacheValue)) { return; } cacheValue.AccessRate = 1; again: var writer = _cacheValueWriter; while (writer == null || writer.GetCurrentPosition() + content.Length > _sizeLimitOfOneValueFile) { StartNewValueFile(); writer = _cacheValueWriter; } lock (writer) { if (writer != _cacheValueWriter) { goto again; } cacheValue.FileId = _cacheValueFileId; cacheValue.FileOfs = (uint)_cacheValueWriter.GetCurrentPosition(); _cacheValueWriter.WriteBlock(content); _cacheValueFile.Flush(); } cacheValue.ContentLength = (uint)content.Length; _cache.TryAdd(k, cacheValue); }
void MoveValuesContent(AbstractBufferedWriter writer, uint wastefullFileId, uint pvlFileId) { const uint blockSize = 256 * 1024; var wasteFullStream = _keyValueDB.FileCollection.GetFile(wastefullFileId); var totalSize = wasteFullStream.GetSize(); var blocks = (int)((totalSize + blockSize - 1) / blockSize); var wasteInMemory = new byte[blocks][]; var pos = 0UL; var readLimiter = new BytesPerSecondLimiter(_keyValueDB._compactorReadBytesPerSecondLimit); for (var i = 0; i < blocks; i++) { _cancellation.ThrowIfCancellationRequested(); wasteInMemory[i] = new byte[blockSize]; var readSize = totalSize - pos; if (readSize > blockSize) { readSize = blockSize; } wasteFullStream.RandomRead(wasteInMemory[i], 0, (int)readSize, pos, true); pos += readSize; readLimiter.Limit(pos); } _root.Iterate((valueFileId, valueOfs, valueSize) => { if (valueFileId != wastefullFileId) { return; } var size = (uint)Math.Abs(valueSize); _newPositionMap.Add(((ulong)wastefullFileId << 32) | valueOfs, ((ulong)pvlFileId << 32) + (ulong)writer.GetCurrentPosition()); pos = valueOfs; while (size > 0) { _cancellation.ThrowIfCancellationRequested(); var blockId = pos / blockSize; var blockStart = pos % blockSize; var writeSize = (uint)(blockSize - blockStart); if (writeSize > size) { writeSize = size; } writer.WriteBlock(wasteInMemory[blockId], (int)blockStart, (int)writeSize); size -= writeSize; pos += writeSize; _writerBytesPerSecondLimiter.Limit((ulong)writer.GetCurrentPosition()); } }); }
public void WriteCreateOrUpdateCommand(byte[] prefix, ByteBuffer key, ByteBuffer value, out uint valueFileId, out uint valueOfs, out int valueSize) { var command = KVCommandType.CreateOrUpdate; if (_compression.ShouldTryToCompressKey(prefix.Length + key.Length)) { if (prefix.Length != 0) { var fullkey = new byte[prefix.Length + key.Length]; Array.Copy(prefix, 0, fullkey, 0, prefix.Length); Array.Copy(key.Buffer, key.Offset, fullkey, prefix.Length, key.Length); prefix = BitArrayManipulation.EmptyByteArray; key = ByteBuffer.NewAsync(fullkey); } if (_compression.CompressKey(ref key)) { command |= KVCommandType.FirstParamCompressed; } } valueSize = value.Length; if (_compression.CompressValue(ref value)) { command |= KVCommandType.SecondParamCompressed; valueSize = -value.Length; } if (_writerWithTransactionLog.GetCurrentPosition() + prefix.Length + key.Length + 16 > MaxTrLogFileSize) { WriteStartOfNewTransactionLogFile(); } _writerWithTransactionLog.WriteUInt8((byte)command); _writerWithTransactionLog.WriteVInt32(prefix.Length + key.Length); _writerWithTransactionLog.WriteVInt32(value.Length); _writerWithTransactionLog.WriteBlock(prefix); _writerWithTransactionLog.WriteBlock(key); if (valueSize != 0) { if (valueSize > 0 && valueSize < MaxValueSizeInlineInMemory) { StoreValueInlineInMemory(value, out valueOfs, out valueSize); valueFileId = 0; } else { valueFileId = _fileIdWithTransactionLog; valueOfs = (uint)_writerWithTransactionLog.GetCurrentPosition(); } _writerWithTransactionLog.WriteBlock(value); } else { valueFileId = 0; valueOfs = 0; } }
public void SetKeyPrefix(byte[] prefix, int prefixOfs, int prefixLen) { lock (_log) { _log.WriteUInt8((byte)KVReplayOperation.SetKeyPrefix); _log.WriteVUInt32(TrIndex); _log.WriteVInt32(prefixLen); _log.WriteVInt32(prefixOfs); _log.WriteBlock(prefix, prefixOfs, prefixLen); _log.FlushBuffer(); } _tr.SetKeyPrefix(prefix, prefixOfs, prefixLen); }
StorageValue StoreContent(ByteBuffer content) { var result = new StorageValue(); result.Compressed = false; result.ContentLength = (uint)content.Length; if (_pureValueFile == null) { StartNewPureValueFile(); } result.FileId = _pureValueFile.Index; result.FileOfs = (uint)_pureValueFileWriter.GetCurrentPosition(); _pureValueFileWriter.WriteBlock(content); if (_pureValueFileWriter.GetCurrentPosition() >= _maxFileSize) { _pureValueFileWriter.FlushBuffer(); StartNewPureValueFile(); } return(result); }
public void WriteRelationPKPrefix(AbstractBufferedWriter writer) { writer.WriteBlock(_relationInfo.Prefix); }
public void WriteRelationSKPrefix(AbstractBufferedWriter writer, uint secondaryKeyIndex) { writer.WriteBlock(_relationInfo.PrefixSecondary); writer.WriteUInt8((byte)secondaryKeyIndex); }