void StoreHashIndex() { RemoveAllHashIndexAndUnknownFiles(); var file = _fileCollection.AddFile("chi"); var writerController = file.GetExclusiveAppenderWriter(); var writer = new SpanWriter(writerController); var keyCount = _cache.Count; var fileInfo = new FileHashIndex(AllocNewFileGeneration(), _keySize, keyCount); _fileInfos.TryAdd(file.Index, fileInfo); fileInfo.WriteHeader(ref writer); var keyBuf = ByteBuffer.NewSync(new byte[_keySize]); foreach (var cachePair in _cache) { cachePair.Key.FillBuffer(keyBuf); writer.WriteVUInt32(cachePair.Value.FileOfs); writer.WriteVUInt32(cachePair.Value.FileId); writer.WriteVUInt32(cachePair.Value.AccessRate); writer.WriteVUInt32(cachePair.Value.ContentLength); writer.WriteBlock(keyBuf); } writer.WriteVUInt32(0); // Zero FileOfs as End of file mark writer.Sync(); file.HardFlushTruncateSwitchToDisposedMode(); }
void StartNewHashIndexFile() { _hashIndexFile = _fileCollection.AddFile("hid"); _hashIndexWriter = _hashIndexFile.GetExclusiveAppenderWriter(); var fileInfo = new HashKeyIndex(_subDBId, _fileCollection.NextGeneration(), _fileCollection.Guid, (uint)_keyLen); var writer = new SpanWriter(_hashIndexWriter); fileInfo.WriteHeader(ref writer); writer.Sync(); _fileCollection.SetInfo(_hashIndexFile.Index, fileInfo); }
void StartNewPureValueFile() { _pureValueFile = _fileCollection.AddFile("hpv"); _pureValueFileWriter = _pureValueFile.GetAppenderWriter(); var fileInfo = new FilePureValuesWithId(_subDBId, _fileCollection.NextGeneration(), _fileCollection.Guid); var writer = new SpanWriter(_pureValueFileWriter); fileInfo.WriteHeader(ref writer); writer.Sync(); _pureValueFile.Flush(); _fileCollection.SetInfo(_pureValueFile.Index, fileInfo); }
void PreserveJustMostOftenUsed(uint fileId) { var frequencies = new List <uint>(); foreach (var itemPair in _cache) { if (itemPair.Value.FileId == fileId) { frequencies.Add(itemPair.Value.AccessRate); } } var preserveRate = frequencies.OrderByDescending(r => r).Skip(frequencies.Count / 5).FirstOrDefault(); foreach (var itemPair in _cache) { if (itemPair.Value.FileId == fileId) { if (preserveRate < itemPair.Value.AccessRate) { var cacheValue = itemPair.Value; var content = new byte[cacheValue.ContentLength]; _fileCollection.GetFile(cacheValue.FileId).RandomRead(content.AsSpan(0, (int)cacheValue.ContentLength), cacheValue.FileOfs, true); var writer = _cacheValueWriter; if (writer == null) { goto remove; } lock (writer) { if (writer != _cacheValueWriter) { goto remove; } if (writer.GetCurrentPositionWithoutWriter() + cacheValue.ContentLength > _sizeLimitOfOneValueFile) { goto remove; } cacheValue.FileId = _cacheValueFileId; cacheValue.FileOfs = (uint)writer.GetCurrentPositionWithoutWriter(); var trueWriter = new SpanWriter(writer); trueWriter.WriteBlock(content); trueWriter.Sync(); } _cache.TryUpdate(itemPair.Key, cacheValue, itemPair.Value); continue; } remove: _cache.TryRemove(itemPair.Key); } } }
public void ContinuousMemoryBlockWriterBasicsWorks() { var byteArrayWriter = new ContinuousMemoryBlockWriter(); Assert.Equal(0, byteArrayWriter.GetCurrentPositionWithoutWriter()); Assert.Equal(Array.Empty <byte>(), byteArrayWriter.GetSpan().ToArray()); Assert.Equal(Array.Empty <byte>(), byteArrayWriter.GetByteBuffer().ToByteArray()); var writer = new SpanWriter(byteArrayWriter); writer.WriteInt8(42); writer.Sync(); Assert.Equal(1, byteArrayWriter.GetCurrentPositionWithoutWriter()); Assert.Equal(new byte[] { 42 }, byteArrayWriter.GetSpan().ToArray()); Assert.Equal(new byte[] { 42 }, byteArrayWriter.GetByteBuffer().ToByteArray()); writer.WriteInt8(1); Assert.Equal(2, writer.GetCurrentPosition()); writer.SetCurrentPosition(1); writer.WriteBlock(new byte[] { 43, 44 }); writer.Sync(); Assert.Equal(3, byteArrayWriter.GetCurrentPositionWithoutWriter()); Assert.Equal(new byte[] { 42, 43, 44 }, byteArrayWriter.GetSpan().ToArray()); Assert.Equal(new byte[] { 42, 43, 44 }, byteArrayWriter.GetByteBuffer().ToByteArray()); }
public void Put(ByteBuffer key, ByteBuffer content) { if (key.Length != _keySize) { throw new ArgumentException("Key has wrong Length not equal to KeySize"); } if (content.Length == 0) { throw new ArgumentException("Empty Content cannot be stored"); } var k = new ByteStructs.Key20(key); if (_cache.TryGetValue(k, out var cacheValue)) { return; } cacheValue.AccessRate = 1; again: var writer = _cacheValueWriter; while (writer == null || writer.GetCurrentPositionWithoutWriter() + content.Length > _sizeLimitOfOneValueFile) { StartNewValueFile(); writer = _cacheValueWriter; } lock (writer) { if (writer != _cacheValueWriter) { goto again; } cacheValue.FileId = _cacheValueFileId; cacheValue.FileOfs = (uint)writer.GetCurrentPositionWithoutWriter(); var trueWriter = new SpanWriter(writer); trueWriter.WriteBlock(content); trueWriter.Sync(); _cacheValueFile !.Flush(); } cacheValue.ContentLength = (uint)content.Length; _cache.TryAdd(k, cacheValue); }
public void CanCreateFile() { using (var dc = new OnDiskFileCollection(_dir)) { var f = dc.AddFile("kvi"); var w = f.GetAppenderWriter(); var writer = new SpanWriter(w); for (var i = 0; i < 32000; i++) { writer.WriteInt32LE(i); } writer.Sync(); var data = new byte[4]; for (var i = 0; i < 32000; i++) { f.RandomRead(data, (ulong)i * 4, false); Assert.Equal(i, PackUnpack.UnpackInt32LE(data, 0)); } } }