void StoreHashIndex()
{
RemoveAllHashIndexAndUnknownFiles();
var file = _fileCollection.AddFile("chi");
var writerController = file.GetExclusiveAppenderWriter();
var writer = new SpanWriter(writerController);
var keyCount = _cache.Count;
var fileInfo = new FileHashIndex(AllocNewFileGeneration(), _keySize, keyCount);
_fileInfos.TryAdd(file.Index, fileInfo);
fileInfo.WriteHeader(ref writer);
var keyBuf = ByteBuffer.NewSync(new byte[_keySize]);
foreach (var cachePair in _cache)
{
cachePair.Key.FillBuffer(keyBuf);
writer.WriteVUInt32(cachePair.Value.FileOfs);
writer.WriteVUInt32(cachePair.Value.FileId);
writer.WriteVUInt32(cachePair.Value.AccessRate);
writer.WriteVUInt32(cachePair.Value.ContentLength);
writer.WriteBlock(keyBuf);
}
writer.WriteVUInt32(0); // Zero FileOfs as End of file mark
writer.Sync();
file.HardFlushTruncateSwitchToDisposedMode();
}
ReadOnlySpan <byte> KeyToByteArray(TKey key, ref SpanWriter writer)
{
writer.WriteBlock(_prefix);
IWriterCtx ctx = null;
if (_keyHandler.NeedsCtx())
{
ctx = new DBWriterCtx(_tr);
}
_keyWriter(key, ref writer, ctx);
return(writer.GetPersistentSpanAndReset());
}
void PreserveJustMostOftenUsed(uint fileId)
{
var frequencies = new List <uint>();
foreach (var itemPair in _cache)
{
if (itemPair.Value.FileId == fileId)
{
frequencies.Add(itemPair.Value.AccessRate);
}
}
var preserveRate = frequencies.OrderByDescending(r => r).Skip(frequencies.Count / 5).FirstOrDefault();
foreach (var itemPair in _cache)
{
if (itemPair.Value.FileId == fileId)
{
if (preserveRate < itemPair.Value.AccessRate)
{
var cacheValue = itemPair.Value;
var content = new byte[cacheValue.ContentLength];
_fileCollection.GetFile(cacheValue.FileId).RandomRead(content.AsSpan(0, (int)cacheValue.ContentLength),
cacheValue.FileOfs, true);
var writer = _cacheValueWriter;
if (writer == null)
{
goto remove;
}
lock (writer)
{
if (writer != _cacheValueWriter)
{
goto remove;
}
if (writer.GetCurrentPositionWithoutWriter() + cacheValue.ContentLength > _sizeLimitOfOneValueFile)
{
goto remove;
}
cacheValue.FileId = _cacheValueFileId;
cacheValue.FileOfs = (uint)writer.GetCurrentPositionWithoutWriter();
var trueWriter = new SpanWriter(writer);
trueWriter.WriteBlock(content);
trueWriter.Sync();
}
_cache.TryUpdate(itemPair.Key, cacheValue, itemPair.Value);
continue;
}
remove:
_cache.TryRemove(itemPair.Key);
}
}
}
ReadOnlySpan <byte> KeyToByteArray(TKey key)
{
var writer = new SpanWriter();
IWriterCtx ctx = null;
if (_keyHandler.NeedsCtx())
{
ctx = new DBWriterCtx(_tr);
}
writer.WriteBlock(_prefix);
_keyWriter(key, ref writer, ctx);
return(writer.GetSpan());
}
public void Put(ByteBuffer key, ByteBuffer content)
{
if (key.Length != _keySize)
{
throw new ArgumentException("Key has wrong Length not equal to KeySize");
}
if (content.Length == 0)
{
throw new ArgumentException("Empty Content cannot be stored");
}
var k = new ByteStructs.Key20(key);
if (_cache.TryGetValue(k, out var cacheValue))
{
return;
}
cacheValue.AccessRate = 1;
again:
var writer = _cacheValueWriter;
while (writer == null || writer.GetCurrentPositionWithoutWriter() + content.Length > _sizeLimitOfOneValueFile)
{
StartNewValueFile();
writer = _cacheValueWriter;
}
lock (writer)
{
if (writer != _cacheValueWriter)
{
goto again;
}
cacheValue.FileId = _cacheValueFileId;
cacheValue.FileOfs = (uint)writer.GetCurrentPositionWithoutWriter();
var trueWriter = new SpanWriter(writer);
trueWriter.WriteBlock(content);
trueWriter.Sync();
_cacheValueFile !.Flush();
}
cacheValue.ContentLength = (uint)content.Length;
_cache.TryAdd(k, cacheValue);
}
public void ContinuousMemoryBlockWriterBasicsWorks()
{
var byteArrayWriter = new ContinuousMemoryBlockWriter();
Assert.Equal(0, byteArrayWriter.GetCurrentPositionWithoutWriter());
Assert.Equal(Array.Empty <byte>(), byteArrayWriter.GetSpan().ToArray());
Assert.Equal(Array.Empty <byte>(), byteArrayWriter.GetByteBuffer().ToByteArray());
var writer = new SpanWriter(byteArrayWriter);
writer.WriteInt8(42);
writer.Sync();
Assert.Equal(1, byteArrayWriter.GetCurrentPositionWithoutWriter());
Assert.Equal(new byte[] { 42 }, byteArrayWriter.GetSpan().ToArray());
Assert.Equal(new byte[] { 42 }, byteArrayWriter.GetByteBuffer().ToByteArray());
writer.WriteInt8(1);
Assert.Equal(2, writer.GetCurrentPosition());
writer.SetCurrentPosition(1);
writer.WriteBlock(new byte[] { 43, 44 });
writer.Sync();
Assert.Equal(3, byteArrayWriter.GetCurrentPositionWithoutWriter());
Assert.Equal(new byte[] { 42, 43, 44 }, byteArrayWriter.GetSpan().ToArray());
Assert.Equal(new byte[] { 42, 43, 44 }, byteArrayWriter.GetByteBuffer().ToByteArray());
}
internal RelationInfo CreateByName(IInternalObjectDBTransaction tr, string name, Type interfaceType,
RelationBuilder builder)
{
name = string.Intern(name);
if (!_name2Id.TryGetValue(name, out var id))
{
id = _freeId++;
_name2Id[name] = id;
var nameWriter = new SpanWriter();
nameWriter.WriteBlock(ObjectDB.RelationNamesPrefix);
nameWriter.WriteString(name);
var idWriter = new SpanWriter();
idWriter.WriteVUInt32(id);
tr.KeyValueDBTransaction.CreateOrUpdateKeyValue(nameWriter.GetSpan(), idWriter.GetSpan());
}
if (_id2Relation.TryGetValue(id, out var relation))
{
throw new BTDBException($"Relation with name '{name}' was already initialized");
}
relation = new RelationInfo(id, name, builder, tr);
_id2Relation[id] = relation;
return(relation);
}
public void WriteRelationSKPrefix(ref SpanWriter writer, uint secondaryKeyIndex)
{
writer.WriteBlock(_relationInfo.PrefixSecondary);
writer.WriteUInt8((byte)secondaryKeyIndex);
}
public void WriteRelationPKPrefix(ref SpanWriter writer)
{
writer.WriteBlock(_relationInfo.Prefix);
}
void SerializeIntoBuffer(object?metadata, IReadOnlyList <object>?events, out int startOffset,
out IDescriptorSerializerContext serializerContext, out BlockType blockType,
out int lenWithoutEndPadding, out ByteBuffer block)
{
startOffset = (int)EndBufferLen + HeaderSize;
var writer = new SpanWriter();
writer.WriteBlock(_zeroes.AsSpan(0, startOffset));
serializerContext = Mapping;
if (metadata != null)
{
serializerContext = serializerContext.StoreNewDescriptors(metadata);
}
if (events != null)
{
foreach (var o in events)
{
serializerContext = serializerContext.StoreNewDescriptors(o);
}
if (events.Count == 0)
{
events = null;
}
}
serializerContext.FinishNewDescriptors(ref writer);
blockType = BlockType.FirstBlock;
if (serializerContext.SomeTypeStored)
{
blockType |= BlockType.HasTypeDeclaration;
}
if (metadata != null)
{
serializerContext.StoreObject(ref writer, metadata);
blockType |= BlockType.HasMetadata;
}
if (events != null)
{
if (events.Count == 1)
{
serializerContext.StoreObject(ref writer, events[0]);
blockType |= BlockType.HasOneEvent;
}
else
{
writer.WriteVUInt32((uint)events.Count);
foreach (var o in events)
{
serializerContext.StoreObject(ref writer, o);
}
blockType |= BlockType.HasMoreEvents;
}
}
lenWithoutEndPadding = (int)writer.GetCurrentPosition();
writer.WriteBlock(_zeroes.AsSpan(0, (int)(SectorSize - 1)));
block = writer.GetByteBufferAndReset();
if (CompressionStrategy.ShouldTryToCompress(lenWithoutEndPadding - startOffset))
{
var compressedBlock = new ReadOnlySpan <byte>(block.Buffer, startOffset, lenWithoutEndPadding - startOffset);
if (CompressionStrategy.Compress(ref compressedBlock))
{
blockType |= BlockType.Compressed;
compressedBlock.CopyTo(new Span <byte>(block.Buffer, startOffset, compressedBlock.Length));
lenWithoutEndPadding = startOffset + compressedBlock.Length;
new Span <byte>(block.Buffer, lenWithoutEndPadding, (int)SectorSize - 1).Clear();
}
}
}
