public bool Open(IPositionLessStream positionLessStream, bool dispose)
{
lock (_log)
{
_log.WriteUInt8((byte)KVReplayOperation.Open);
ulong size = positionLessStream.GetSize();
_log.WriteVUInt64(size);
ulong pos = 0;
var buf = new byte[4096];
while (pos < size)
{
var read = positionLessStream.Read(buf, 0, buf.Length, pos);
// Next 2 conditions should not happen or file is mutated when it should not
if (read == 0)
{
break;
}
if ((ulong)read > size - pos)
{
read = (int)(size - pos);
}
_log.WriteBlock(buf, 0, read);
pos += (ulong)read;
}
while (pos < size)
{
_log.WriteUInt8(0);
pos++;
}
_log.FlushBuffer();
}
return(_db.Open(positionLessStream, dispose));
}
void StoreHashUpdate(ByteBuffer key, StorageValue storageValue)
{
if (_hashIndexWriter == null)
{
StartNewHashIndexFile();
}
_hashIndexWriter.WriteVUInt32(storageValue.FileId);
_hashIndexWriter.WriteVUInt32(storageValue.FileOfs);
_hashIndexWriter.WriteVUInt32(storageValue.ContentLengthCompressedIsLeaf);
_hashIndexWriter.WriteBlock(key);
}
public void Put(ByteBuffer key, ByteBuffer content)
{
if (key.Length != _keySize)
{
throw new ArgumentException("Key has wrong Length not equal to KeySize");
}
if (content.Length == 0)
{
throw new ArgumentException("Empty Content cannot be stored");
}
var k = new ByteStructs.Key20(key);
CacheValue cacheValue;
if (_cache.TryGetValue(k, out cacheValue))
{
return;
}
cacheValue.AccessRate = 1;
again:
var writer = _cacheValueWriter;
while (writer == null || writer.GetCurrentPosition() + content.Length > _sizeLimitOfOneValueFile)
{
StartNewValueFile();
writer = _cacheValueWriter;
}
lock (writer)
{
if (writer != _cacheValueWriter)
{
goto again;
}
cacheValue.FileId = _cacheValueFileId;
cacheValue.FileOfs = (uint)_cacheValueWriter.GetCurrentPosition();
_cacheValueWriter.WriteBlock(content);
_cacheValueFile.Flush();
}
cacheValue.ContentLength = (uint)content.Length;
_cache.TryAdd(k, cacheValue);
}
void MoveValuesContent(AbstractBufferedWriter writer, uint wastefullFileId, uint pvlFileId)
{
const uint blockSize = 256 * 1024;
var wasteFullStream = _keyValueDB.FileCollection.GetFile(wastefullFileId);
var totalSize = wasteFullStream.GetSize();
var blocks = (int)((totalSize + blockSize - 1) / blockSize);
var wasteInMemory = new byte[blocks][];
var pos = 0UL;
var readLimiter = new BytesPerSecondLimiter(_keyValueDB._compactorReadBytesPerSecondLimit);
for (var i = 0; i < blocks; i++)
{
_cancellation.ThrowIfCancellationRequested();
wasteInMemory[i] = new byte[blockSize];
var readSize = totalSize - pos;
if (readSize > blockSize)
{
readSize = blockSize;
}
wasteFullStream.RandomRead(wasteInMemory[i], 0, (int)readSize, pos, true);
pos += readSize;
readLimiter.Limit(pos);
}
_root.Iterate((valueFileId, valueOfs, valueSize) =>
{
if (valueFileId != wastefullFileId)
{
return;
}
var size = (uint)Math.Abs(valueSize);
_newPositionMap.Add(((ulong)wastefullFileId << 32) | valueOfs,
((ulong)pvlFileId << 32) + (ulong)writer.GetCurrentPosition());
pos = valueOfs;
while (size > 0)
{
_cancellation.ThrowIfCancellationRequested();
var blockId = pos / blockSize;
var blockStart = pos % blockSize;
var writeSize = (uint)(blockSize - blockStart);
if (writeSize > size)
{
writeSize = size;
}
writer.WriteBlock(wasteInMemory[blockId], (int)blockStart, (int)writeSize);
size -= writeSize;
pos += writeSize;
_writerBytesPerSecondLimiter.Limit((ulong)writer.GetCurrentPosition());
}
});
}
public void WriteCreateOrUpdateCommand(byte[] prefix, ByteBuffer key, ByteBuffer value, out uint valueFileId, out uint valueOfs, out int valueSize)
{
var command = KVCommandType.CreateOrUpdate;
if (_compression.ShouldTryToCompressKey(prefix.Length + key.Length))
{
if (prefix.Length != 0)
{
var fullkey = new byte[prefix.Length + key.Length];
Array.Copy(prefix, 0, fullkey, 0, prefix.Length);
Array.Copy(key.Buffer, key.Offset, fullkey, prefix.Length, key.Length);
prefix = BitArrayManipulation.EmptyByteArray;
key = ByteBuffer.NewAsync(fullkey);
}
if (_compression.CompressKey(ref key))
{
command |= KVCommandType.FirstParamCompressed;
}
}
valueSize = value.Length;
if (_compression.CompressValue(ref value))
{
command |= KVCommandType.SecondParamCompressed;
valueSize = -value.Length;
}
if (_writerWithTransactionLog.GetCurrentPosition() + prefix.Length + key.Length + 16 > MaxTrLogFileSize)
{
WriteStartOfNewTransactionLogFile();
}
_writerWithTransactionLog.WriteUInt8((byte)command);
_writerWithTransactionLog.WriteVInt32(prefix.Length + key.Length);
_writerWithTransactionLog.WriteVInt32(value.Length);
_writerWithTransactionLog.WriteBlock(prefix);
_writerWithTransactionLog.WriteBlock(key);
if (valueSize != 0)
{
if (valueSize > 0 && valueSize < MaxValueSizeInlineInMemory)
{
StoreValueInlineInMemory(value, out valueOfs, out valueSize);
valueFileId = 0;
}
else
{
valueFileId = _fileIdWithTransactionLog;
valueOfs = (uint)_writerWithTransactionLog.GetCurrentPosition();
}
_writerWithTransactionLog.WriteBlock(value);
}
else
{
valueFileId = 0;
valueOfs = 0;
}
}
public void SetKeyPrefix(byte[] prefix, int prefixOfs, int prefixLen)
{
lock (_log)
{
_log.WriteUInt8((byte)KVReplayOperation.SetKeyPrefix);
_log.WriteVUInt32(TrIndex);
_log.WriteVInt32(prefixLen);
_log.WriteVInt32(prefixOfs);
_log.WriteBlock(prefix, prefixOfs, prefixLen);
_log.FlushBuffer();
}
_tr.SetKeyPrefix(prefix, prefixOfs, prefixLen);
}
StorageValue StoreContent(ByteBuffer content)
{
var result = new StorageValue();
result.Compressed = false;
result.ContentLength = (uint)content.Length;
if (_pureValueFile == null)
{
StartNewPureValueFile();
}
result.FileId = _pureValueFile.Index;
result.FileOfs = (uint)_pureValueFileWriter.GetCurrentPosition();
_pureValueFileWriter.WriteBlock(content);
if (_pureValueFileWriter.GetCurrentPosition() >= _maxFileSize)
{
_pureValueFileWriter.FlushBuffer();
StartNewPureValueFile();
}
return(result);
}
public void WriteRelationPKPrefix(AbstractBufferedWriter writer)
{
writer.WriteBlock(_relationInfo.Prefix);
}
public void WriteRelationSKPrefix(AbstractBufferedWriter writer, uint secondaryKeyIndex)
{
writer.WriteBlock(_relationInfo.PrefixSecondary);
writer.WriteUInt8((byte)secondaryKeyIndex);
}
