uint CreateKeyIndexFile(IBTreeRootNode root, CancellationToken cancellation)
{
var start = DateTime.UtcNow;
var file = FileCollection.AddFile("kvi");
var writer = file.GetAppenderWriter();
var keyCount = root.CalcKeyCount();
if (root.TrLogFileId != 0)
{
FileCollection.ConcurentTemporaryTruncate(root.TrLogFileId, root.TrLogOffset);
}
var keyIndex = new FileKeyIndex(FileCollection.NextGeneration(), FileCollection.Guid, root.TrLogFileId, root.TrLogOffset, keyCount, root.CommitUlong, KeyIndexCompression.None);
keyIndex.WriteHeader(writer);
if (keyCount > 0)
{
var stack = new List <NodeIdxPair>();
var prevKey = ByteBuffer.NewEmpty();
root.FillStackByIndex(stack, 0);
do
{
cancellation.ThrowIfCancellationRequested();
var nodeIdxPair = stack[stack.Count - 1];
var memberValue = ((IBTreeLeafNode)nodeIdxPair.Node).GetMemberValue(nodeIdxPair.Idx);
var key = ((IBTreeLeafNode)nodeIdxPair.Node).GetKey(nodeIdxPair.Idx);
var prefixLen = 0;
var minLen = Math.Min(prevKey.Length, key.Length);
for (int i = 0; i < minLen; i++)
{
if (prevKey[i] != key[i])
{
prefixLen = i;
break;
}
}
writer.WriteVUInt32((uint)prefixLen);
writer.WriteVUInt32((uint)(key.Length - prefixLen));
writer.WriteBlock(key.SubBuffer(prefixLen));
writer.WriteVUInt32(memberValue.ValueFileId);
writer.WriteVUInt32(memberValue.ValueOfs);
writer.WriteVInt32(memberValue.ValueSize);
prevKey = key;
} while (root.FindNextKey(stack));
}
writer.FlushBuffer();
file.HardFlush();
writer.WriteInt32(EndOfIndexFileMarker);
writer.FlushBuffer();
file.HardFlush();
file.Truncate();
FileCollection.SetInfo(file.Index, keyIndex);
Logger?.KeyValueIndexCreated(file.Index, keyIndex.KeyValueCount, file.GetSize(), DateTime.UtcNow - start);
return(file.Index);
}
void LoadInfoAboutFiles()
{
foreach (var file in _fileCollection.Enumerate())
{
try
{
var reader = file.GetExclusiveReader();
if (reader.CheckMagic(MagicStartOfFile))
{
var fileType = (KVFileType)reader.ReadUInt8();
IFileInfo fileInfo;
switch (fileType)
{
case KVFileType.TransactionLog:
fileInfo = new FileTransactionLog(reader);
break;
case KVFileType.KeyIndex:
fileInfo = new FileKeyIndex(reader);
break;
case KVFileType.PureValues:
fileInfo = new FilePureValues(reader);
break;
case KVFileType.PureValuesWithId:
fileInfo = new FilePureValuesWithId(reader);
break;
case KVFileType.HashKeyIndex:
fileInfo = new HashKeyIndex(reader);
break;
default:
fileInfo = UnknownFile.Instance;
break;
}
if (_fileGeneration < fileInfo.Generation) _fileGeneration = fileInfo.Generation;
_fileInfos.TryAdd(file.Index, fileInfo);
}
else
{
_fileInfos.TryAdd(file.Index, UnknownFile.Instance);
}
}
catch (Exception)
{
_fileInfos.TryAdd(file.Index, UnknownFile.Instance);
}
}
}
void LoadInfoAboutFiles()
{
foreach (var file in _fileCollection.Enumerate())
{
try
{
var reader = file.GetExclusiveReader();
var magic = reader.ReadByteArrayRaw(MagicStartOfFile.Length);
Guid?guid = null;
if (
BitArrayManipulation.CompareByteArray(magic, magic.Length, MagicStartOfFileWithGuid,
MagicStartOfFileWithGuid.Length) == 0)
{
guid = reader.ReadGuid();
if (Guid.HasValue && Guid.Value != guid)
{
_fileInfos.TryAdd(file.Index, UnknownFile.Instance);
continue;
}
Guid = guid;
}
else if (
BitArrayManipulation.CompareByteArray(magic, magic.Length, MagicStartOfFile,
MagicStartOfFile.Length) != 0)
{
_fileInfos.TryAdd(file.Index, UnknownFile.Instance);
continue;
}
var fileType = (KVFileType)reader.ReadUInt8();
IFileInfo fileInfo;
switch (fileType)
{
case KVFileType.TransactionLog:
fileInfo = new FileTransactionLog(reader, guid);
break;
case KVFileType.KeyIndex:
fileInfo = new FileKeyIndex(reader, guid, false, false, false);
break;
case KVFileType.KeyIndexWithCommitUlong:
fileInfo = new FileKeyIndex(reader, guid, true, false, false);
break;
case KVFileType.ModernKeyIndex:
fileInfo = new FileKeyIndex(reader, guid, true, true, false);
break;
case KVFileType.ModernKeyIndexWithUlongs:
fileInfo = new FileKeyIndex(reader, guid, true, true, true);
break;
case KVFileType.PureValues:
fileInfo = new FilePureValues(reader, guid);
break;
case KVFileType.PureValuesWithId:
fileInfo = new FilePureValuesWithId(reader, guid);
break;
case KVFileType.HashKeyIndex:
fileInfo = new HashKeyIndex(reader, guid);
break;
default:
fileInfo = UnknownFile.Instance;
break;
}
if (_fileGeneration < fileInfo.Generation)
{
_fileGeneration = fileInfo.Generation;
}
_fileInfos.TryAdd(file.Index, fileInfo);
}
catch (Exception)
{
_fileInfos.TryAdd(file.Index, UnknownFile.Instance);
}
}
if (!Guid.HasValue)
{
Guid = System.Guid.NewGuid();
}
}
void LoadInfoAboutFiles()
{
foreach (var file in _fileCollection.Enumerate())
{
try
{
var reader = file.GetExclusiveReader();
var magic = reader.ReadByteArrayRaw(MagicStartOfFile.Length);
Guid? guid = null;
if (
BitArrayManipulation.CompareByteArray(magic, magic.Length, MagicStartOfFileWithGuid,
MagicStartOfFileWithGuid.Length) == 0)
{
guid = reader.ReadGuid();
if (Guid.HasValue && Guid.Value != guid)
{
_fileInfos.TryAdd(file.Index, UnknownFile.Instance);
continue;
}
Guid = guid;
}
else if (
BitArrayManipulation.CompareByteArray(magic, magic.Length, MagicStartOfFile,
MagicStartOfFile.Length) != 0)
{
_fileInfos.TryAdd(file.Index, UnknownFile.Instance);
continue;
}
var fileType = (KVFileType)reader.ReadUInt8();
IFileInfo fileInfo;
switch (fileType)
{
case KVFileType.TransactionLog:
fileInfo = new FileTransactionLog(reader, guid);
break;
case KVFileType.KeyIndex:
fileInfo = new FileKeyIndex(reader, guid, false);
break;
case KVFileType.KeyIndexWithCommitUlong:
fileInfo = new FileKeyIndex(reader, guid, true);
break;
case KVFileType.PureValues:
fileInfo = new FilePureValues(reader, guid);
break;
case KVFileType.PureValuesWithId:
fileInfo = new FilePureValuesWithId(reader, guid);
break;
case KVFileType.HashKeyIndex:
fileInfo = new HashKeyIndex(reader, guid);
break;
default:
fileInfo = UnknownFile.Instance;
break;
}
if (_fileGeneration < fileInfo.Generation) _fileGeneration = fileInfo.Generation;
_fileInfos.TryAdd(file.Index, fileInfo);
}
catch (Exception)
{
_fileInfos.TryAdd(file.Index, UnknownFile.Instance);
}
}
if (!Guid.HasValue)
{
Guid = System.Guid.NewGuid();
}
}
bool LoadKeyIndex(uint fileId, IKeyIndex info)
{
try
{
var reader = FileCollection.GetFile(fileId).GetExclusiveReader();
FileKeyIndex.SkipHeader(reader);
var keyCount = info.KeyValueCount;
_nextRoot.TrLogFileId = info.TrLogFileId;
_nextRoot.TrLogOffset = info.TrLogOffset;
_nextRoot.CommitUlong = info.CommitUlong;
if (info.Compression == KeyIndexCompression.Old)
{
_nextRoot.BuildTree(keyCount, () =>
{
var keyLength = reader.ReadVInt32();
var key = ByteBuffer.NewAsync(new byte[Math.Abs(keyLength)]);
reader.ReadBlock(key);
if (keyLength < 0)
{
_compression.DecompressKey(ref key);
}
return(new BTreeLeafMember
{
Key = key.ToByteArray(),
ValueFileId = reader.ReadVUInt32(),
ValueOfs = reader.ReadVUInt32(),
ValueSize = reader.ReadVInt32()
});
});
}
else
{
if (info.Compression != KeyIndexCompression.None)
{
return(false);
}
var prevKey = ByteBuffer.NewEmpty();
_nextRoot.BuildTree(keyCount, () =>
{
var prefixLen = (int)reader.ReadVUInt32();
var keyLengthWithoutPrefix = (int)reader.ReadVUInt32();
var key = ByteBuffer.NewAsync(new byte[prefixLen + keyLengthWithoutPrefix]);
Array.Copy(prevKey.Buffer, prevKey.Offset, key.Buffer, key.Offset, prefixLen);
reader.ReadBlock(key.SubBuffer(prefixLen));
prevKey = key;
return(new BTreeLeafMember
{
Key = key.ToByteArray(),
ValueFileId = reader.ReadVUInt32(),
ValueOfs = reader.ReadVUInt32(),
ValueSize = reader.ReadVInt32()
});
});
}
if (reader.Eof)
{
return(true);
}
if (reader.ReadInt32() == EndOfIndexFileMarker)
{
return(true);
}
return(false);
}
catch (Exception)
{
return(false);
}
}
