void StoreHashIndex()
{
RemoveAllHashIndexAndUnknownFiles();
var file = _fileCollection.AddFile("chi");
var writerController = file.GetExclusiveAppenderWriter();
var writer = new SpanWriter(writerController);
var keyCount = _cache.Count;
var fileInfo = new FileHashIndex(AllocNewFileGeneration(), _keySize, keyCount);
_fileInfos.TryAdd(file.Index, fileInfo);
fileInfo.WriteHeader(ref writer);
var keyBuf = ByteBuffer.NewSync(new byte[_keySize]);
foreach (var cachePair in _cache)
{
cachePair.Key.FillBuffer(keyBuf);
writer.WriteVUInt32(cachePair.Value.FileOfs);
writer.WriteVUInt32(cachePair.Value.FileId);
writer.WriteVUInt32(cachePair.Value.AccessRate);
writer.WriteVUInt32(cachePair.Value.ContentLength);
writer.WriteBlock(keyBuf);
}
writer.WriteVUInt32(0); // Zero FileOfs as End of file mark
writer.Sync();
file.HardFlushTruncateSwitchToDisposedMode();
}
void StartNewHashIndexFile()
{
_hashIndexFile = _fileCollection.AddFile("hid");
_hashIndexWriter = _hashIndexFile.GetExclusiveAppenderWriter();
var fileInfo = new HashKeyIndex(_subDBId, _fileCollection.NextGeneration(), _fileCollection.Guid, (uint)_keyLen);
var writer = new SpanWriter(_hashIndexWriter);
fileInfo.WriteHeader(ref writer);
writer.Sync();
_fileCollection.SetInfo(_hashIndexFile.Index, fileInfo);
}
void StartNewPureValueFile()
{
_pureValueFile = _fileCollection.AddFile("hpv");
_pureValueFileWriter = _pureValueFile.GetAppenderWriter();
var fileInfo = new FilePureValuesWithId(_subDBId, _fileCollection.NextGeneration(), _fileCollection.Guid);
var writer = new SpanWriter(_pureValueFileWriter);
fileInfo.WriteHeader(ref writer);
writer.Sync();
_pureValueFile.Flush();
_fileCollection.SetInfo(_pureValueFile.Index, fileInfo);
}
void PreserveJustMostOftenUsed(uint fileId)
{
var frequencies = new List <uint>();
foreach (var itemPair in _cache)
{
if (itemPair.Value.FileId == fileId)
{
frequencies.Add(itemPair.Value.AccessRate);
}
}
var preserveRate = frequencies.OrderByDescending(r => r).Skip(frequencies.Count / 5).FirstOrDefault();
foreach (var itemPair in _cache)
{
if (itemPair.Value.FileId == fileId)
{
if (preserveRate < itemPair.Value.AccessRate)
{
var cacheValue = itemPair.Value;
var content = new byte[cacheValue.ContentLength];
_fileCollection.GetFile(cacheValue.FileId).RandomRead(content.AsSpan(0, (int)cacheValue.ContentLength),
cacheValue.FileOfs, true);
var writer = _cacheValueWriter;
if (writer == null)
{
goto remove;
}
lock (writer)
{
if (writer != _cacheValueWriter)
{
goto remove;
}
if (writer.GetCurrentPositionWithoutWriter() + cacheValue.ContentLength > _sizeLimitOfOneValueFile)
{
goto remove;
}
cacheValue.FileId = _cacheValueFileId;
cacheValue.FileOfs = (uint)writer.GetCurrentPositionWithoutWriter();
var trueWriter = new SpanWriter(writer);
trueWriter.WriteBlock(content);
trueWriter.Sync();
}
_cache.TryUpdate(itemPair.Key, cacheValue, itemPair.Value);
continue;
}
remove:
_cache.TryRemove(itemPair.Key);
}
}
}
public void ContinuousMemoryBlockWriterBasicsWorks()
{
var byteArrayWriter = new ContinuousMemoryBlockWriter();
Assert.Equal(0, byteArrayWriter.GetCurrentPositionWithoutWriter());
Assert.Equal(Array.Empty <byte>(), byteArrayWriter.GetSpan().ToArray());
Assert.Equal(Array.Empty <byte>(), byteArrayWriter.GetByteBuffer().ToByteArray());
var writer = new SpanWriter(byteArrayWriter);
writer.WriteInt8(42);
writer.Sync();
Assert.Equal(1, byteArrayWriter.GetCurrentPositionWithoutWriter());
Assert.Equal(new byte[] { 42 }, byteArrayWriter.GetSpan().ToArray());
Assert.Equal(new byte[] { 42 }, byteArrayWriter.GetByteBuffer().ToByteArray());
writer.WriteInt8(1);
Assert.Equal(2, writer.GetCurrentPosition());
writer.SetCurrentPosition(1);
writer.WriteBlock(new byte[] { 43, 44 });
writer.Sync();
Assert.Equal(3, byteArrayWriter.GetCurrentPositionWithoutWriter());
Assert.Equal(new byte[] { 42, 43, 44 }, byteArrayWriter.GetSpan().ToArray());
Assert.Equal(new byte[] { 42, 43, 44 }, byteArrayWriter.GetByteBuffer().ToByteArray());
}
public void Put(ByteBuffer key, ByteBuffer content)
{
if (key.Length != _keySize)
{
throw new ArgumentException("Key has wrong Length not equal to KeySize");
}
if (content.Length == 0)
{
throw new ArgumentException("Empty Content cannot be stored");
}
var k = new ByteStructs.Key20(key);
if (_cache.TryGetValue(k, out var cacheValue))
{
return;
}
cacheValue.AccessRate = 1;
again:
var writer = _cacheValueWriter;
while (writer == null || writer.GetCurrentPositionWithoutWriter() + content.Length > _sizeLimitOfOneValueFile)
{
StartNewValueFile();
writer = _cacheValueWriter;
}
lock (writer)
{
if (writer != _cacheValueWriter)
{
goto again;
}
cacheValue.FileId = _cacheValueFileId;
cacheValue.FileOfs = (uint)writer.GetCurrentPositionWithoutWriter();
var trueWriter = new SpanWriter(writer);
trueWriter.WriteBlock(content);
trueWriter.Sync();
_cacheValueFile !.Flush();
}
cacheValue.ContentLength = (uint)content.Length;
_cache.TryAdd(k, cacheValue);
}
public void CanCreateFile()
{
using (var dc = new OnDiskFileCollection(_dir))
{
var f = dc.AddFile("kvi");
var w = f.GetAppenderWriter();
var writer = new SpanWriter(w);
for (var i = 0; i < 32000; i++)
{
writer.WriteInt32LE(i);
}
writer.Sync();
var data = new byte[4];
for (var i = 0; i < 32000; i++)
{
f.RandomRead(data, (ulong)i * 4, false);
Assert.Equal(i, PackUnpack.UnpackInt32LE(data, 0));
}
}
}
