void StoreHashIndex() { RemoveAllHashIndexAndUnknownFiles(); var file = _fileCollection.AddFile("chi"); var writerController = file.GetExclusiveAppenderWriter(); var writer = new SpanWriter(writerController); var keyCount = _cache.Count; var fileInfo = new FileHashIndex(AllocNewFileGeneration(), _keySize, keyCount); _fileInfos.TryAdd(file.Index, fileInfo); fileInfo.WriteHeader(ref writer); var keyBuf = ByteBuffer.NewSync(new byte[_keySize]); foreach (var cachePair in _cache) { cachePair.Key.FillBuffer(keyBuf); writer.WriteVUInt32(cachePair.Value.FileOfs); writer.WriteVUInt32(cachePair.Value.FileId); writer.WriteVUInt32(cachePair.Value.AccessRate); writer.WriteVUInt32(cachePair.Value.ContentLength); writer.WriteBlock(keyBuf); } writer.WriteVUInt32(0); // Zero FileOfs as End of file mark writer.Sync(); file.HardFlushTruncateSwitchToDisposedMode(); }
ReadOnlySpan <byte> KeyToByteArray(TKey key, ref SpanWriter writer) { writer.WriteBlock(_prefix); IWriterCtx ctx = null; if (_keyHandler.NeedsCtx()) { ctx = new DBWriterCtx(_tr); } _keyWriter(key, ref writer, ctx); return(writer.GetPersistentSpanAndReset()); }
void PreserveJustMostOftenUsed(uint fileId) { var frequencies = new List <uint>(); foreach (var itemPair in _cache) { if (itemPair.Value.FileId == fileId) { frequencies.Add(itemPair.Value.AccessRate); } } var preserveRate = frequencies.OrderByDescending(r => r).Skip(frequencies.Count / 5).FirstOrDefault(); foreach (var itemPair in _cache) { if (itemPair.Value.FileId == fileId) { if (preserveRate < itemPair.Value.AccessRate) { var cacheValue = itemPair.Value; var content = new byte[cacheValue.ContentLength]; _fileCollection.GetFile(cacheValue.FileId).RandomRead(content.AsSpan(0, (int)cacheValue.ContentLength), cacheValue.FileOfs, true); var writer = _cacheValueWriter; if (writer == null) { goto remove; } lock (writer) { if (writer != _cacheValueWriter) { goto remove; } if (writer.GetCurrentPositionWithoutWriter() + cacheValue.ContentLength > _sizeLimitOfOneValueFile) { goto remove; } cacheValue.FileId = _cacheValueFileId; cacheValue.FileOfs = (uint)writer.GetCurrentPositionWithoutWriter(); var trueWriter = new SpanWriter(writer); trueWriter.WriteBlock(content); trueWriter.Sync(); } _cache.TryUpdate(itemPair.Key, cacheValue, itemPair.Value); continue; } remove: _cache.TryRemove(itemPair.Key); } } }
ReadOnlySpan <byte> KeyToByteArray(TKey key) { var writer = new SpanWriter(); IWriterCtx ctx = null; if (_keyHandler.NeedsCtx()) { ctx = new DBWriterCtx(_tr); } writer.WriteBlock(_prefix); _keyWriter(key, ref writer, ctx); return(writer.GetSpan()); }
public void Put(ByteBuffer key, ByteBuffer content) { if (key.Length != _keySize) { throw new ArgumentException("Key has wrong Length not equal to KeySize"); } if (content.Length == 0) { throw new ArgumentException("Empty Content cannot be stored"); } var k = new ByteStructs.Key20(key); if (_cache.TryGetValue(k, out var cacheValue)) { return; } cacheValue.AccessRate = 1; again: var writer = _cacheValueWriter; while (writer == null || writer.GetCurrentPositionWithoutWriter() + content.Length > _sizeLimitOfOneValueFile) { StartNewValueFile(); writer = _cacheValueWriter; } lock (writer) { if (writer != _cacheValueWriter) { goto again; } cacheValue.FileId = _cacheValueFileId; cacheValue.FileOfs = (uint)writer.GetCurrentPositionWithoutWriter(); var trueWriter = new SpanWriter(writer); trueWriter.WriteBlock(content); trueWriter.Sync(); _cacheValueFile !.Flush(); } cacheValue.ContentLength = (uint)content.Length; _cache.TryAdd(k, cacheValue); }
public void ContinuousMemoryBlockWriterBasicsWorks() { var byteArrayWriter = new ContinuousMemoryBlockWriter(); Assert.Equal(0, byteArrayWriter.GetCurrentPositionWithoutWriter()); Assert.Equal(Array.Empty <byte>(), byteArrayWriter.GetSpan().ToArray()); Assert.Equal(Array.Empty <byte>(), byteArrayWriter.GetByteBuffer().ToByteArray()); var writer = new SpanWriter(byteArrayWriter); writer.WriteInt8(42); writer.Sync(); Assert.Equal(1, byteArrayWriter.GetCurrentPositionWithoutWriter()); Assert.Equal(new byte[] { 42 }, byteArrayWriter.GetSpan().ToArray()); Assert.Equal(new byte[] { 42 }, byteArrayWriter.GetByteBuffer().ToByteArray()); writer.WriteInt8(1); Assert.Equal(2, writer.GetCurrentPosition()); writer.SetCurrentPosition(1); writer.WriteBlock(new byte[] { 43, 44 }); writer.Sync(); Assert.Equal(3, byteArrayWriter.GetCurrentPositionWithoutWriter()); Assert.Equal(new byte[] { 42, 43, 44 }, byteArrayWriter.GetSpan().ToArray()); Assert.Equal(new byte[] { 42, 43, 44 }, byteArrayWriter.GetByteBuffer().ToByteArray()); }
internal RelationInfo CreateByName(IInternalObjectDBTransaction tr, string name, Type interfaceType, RelationBuilder builder) { name = string.Intern(name); if (!_name2Id.TryGetValue(name, out var id)) { id = _freeId++; _name2Id[name] = id; var nameWriter = new SpanWriter(); nameWriter.WriteBlock(ObjectDB.RelationNamesPrefix); nameWriter.WriteString(name); var idWriter = new SpanWriter(); idWriter.WriteVUInt32(id); tr.KeyValueDBTransaction.CreateOrUpdateKeyValue(nameWriter.GetSpan(), idWriter.GetSpan()); } if (_id2Relation.TryGetValue(id, out var relation)) { throw new BTDBException($"Relation with name '{name}' was already initialized"); } relation = new RelationInfo(id, name, builder, tr); _id2Relation[id] = relation; return(relation); }
public void WriteRelationSKPrefix(ref SpanWriter writer, uint secondaryKeyIndex) { writer.WriteBlock(_relationInfo.PrefixSecondary); writer.WriteUInt8((byte)secondaryKeyIndex); }
public void WriteRelationPKPrefix(ref SpanWriter writer) { writer.WriteBlock(_relationInfo.Prefix); }
void SerializeIntoBuffer(object?metadata, IReadOnlyList <object>?events, out int startOffset, out IDescriptorSerializerContext serializerContext, out BlockType blockType, out int lenWithoutEndPadding, out ByteBuffer block) { startOffset = (int)EndBufferLen + HeaderSize; var writer = new SpanWriter(); writer.WriteBlock(_zeroes.AsSpan(0, startOffset)); serializerContext = Mapping; if (metadata != null) { serializerContext = serializerContext.StoreNewDescriptors(metadata); } if (events != null) { foreach (var o in events) { serializerContext = serializerContext.StoreNewDescriptors(o); } if (events.Count == 0) { events = null; } } serializerContext.FinishNewDescriptors(ref writer); blockType = BlockType.FirstBlock; if (serializerContext.SomeTypeStored) { blockType |= BlockType.HasTypeDeclaration; } if (metadata != null) { serializerContext.StoreObject(ref writer, metadata); blockType |= BlockType.HasMetadata; } if (events != null) { if (events.Count == 1) { serializerContext.StoreObject(ref writer, events[0]); blockType |= BlockType.HasOneEvent; } else { writer.WriteVUInt32((uint)events.Count); foreach (var o in events) { serializerContext.StoreObject(ref writer, o); } blockType |= BlockType.HasMoreEvents; } } lenWithoutEndPadding = (int)writer.GetCurrentPosition(); writer.WriteBlock(_zeroes.AsSpan(0, (int)(SectorSize - 1))); block = writer.GetByteBufferAndReset(); if (CompressionStrategy.ShouldTryToCompress(lenWithoutEndPadding - startOffset)) { var compressedBlock = new ReadOnlySpan <byte>(block.Buffer, startOffset, lenWithoutEndPadding - startOffset); if (CompressionStrategy.Compress(ref compressedBlock)) { blockType |= BlockType.Compressed; compressedBlock.CopyTo(new Span <byte>(block.Buffer, startOffset, compressedBlock.Length)); lenWithoutEndPadding = startOffset + compressedBlock.Length; new Span <byte>(block.Buffer, lenWithoutEndPadding, (int)SectorSize - 1).Clear(); } } }