public CbKeyFixed(string keyFile, ushort keyLength = 16, bool createNew = false)
{
_ulongSize = sizeof(ulong);
HeaderLength = _ulongSize;
_keyStart = HeaderLength;
_keyLength = keyLength;
_file = keyFile;
if (File.Exists(_file) && createNew)
{
File.Delete(_file);
}
if (File.Exists(addrFile) && createNew)
{
File.Delete(addrFile);
}
FileStream = File.Open(_file, FileMode.OpenOrCreate);
AddrsFileStream = File.Open(addrFile, FileMode.OpenOrCreate);
_reader = new BinaryReader(FileStream);
_readBuffer = new byte[_keyLength * 10000];
_readBufferHandle = GCHandle.Alloc(_readBuffer, GCHandleType.Pinned);
_readBufferPtr = (byte *)_readBufferHandle.AddrOfPinnedObject();
_readBinBuffer = new HashBin(new byte[_keyLength], false);
_readBinBuffer.SetFromPartialArray(_readBuffer, 0, _keyLength, false);
_addrReader = new BinaryReader(AddrsFileStream);
if (FileStream.Length > 8)
{
_count = _reader.ReadUInt64();
_keyEnd = _count * _keyLength;
}
}
public byte[] GetValue(HashBin key)
{
var hint = CbIndex.GetAddressHintForKey(key);
bool found = CbKey.GetKeyDataAddr(key, out var addr, hint, CbIndex.IndexKeyLen);
if (found)
{
return(CbData.GetValue(addr));
}
return(null);
}
public bool HasKey(HashBin key)
{
var hint = CbIndex.GetAddressHintForKey(key);
return(CbKey.HasKey(key, out var unused, false, hint, CbIndex.IndexKeyLen));
}
public KeyHint GetAddressHintForKey(HashBin key)
{
return(GetAddressHintForKey(key.Hash));
}
public int GetKeyIndexFromKey(HashBin key)
{
return(GetKeyIndexFromKey(key.Hash));
}
public bool HasKey(HashBin key, out DataAddr dataAddr, bool getDataAddr = false, KeyHint hint = default(KeyHint), byte skipBytes = 0)
{
if (key.Length != _keyLength)
{
throw new ArgumentException($"Key must be {_keyLength} bytes");
}
// bool hasHint = hint.StartAddr >= _keyStart &&
// hint.EndAddr >= hint.StartAddr
// && hint.EndAddr <= _keyEnd;
int compareLength = _keyLength - skipBytes;
if (!(hint.StartAddr >= _keyStart &&
hint.EndAddr >= hint.StartAddr &&
hint.EndAddr <= _keyEnd))
{
hint.StartAddr = _keyStart;
hint.EndAddr = _keyEnd;
compareLength = _keyLength;
}
FileStream.Position = (long)hint.StartAddr;
var remaining = (long)(hint.EndAddr - hint.StartAddr + _keyLength);
long lastRead;
int amountToRead;
var bufReadPos = 0;
int compareRes;
fixed(byte *ptrKey = &key.Hash[0])
{
while (remaining > 0)
{
amountToRead = (int)(remaining < _readBuffer.Length ? remaining : _readBuffer.Length);
lastRead = FileStream.Read(_readBuffer, 0, amountToRead);
remaining -= lastRead;
bufReadPos = 0;
//loop until buffer read
while (bufReadPos < lastRead)
{
//_readBinBuffer.SetPointer(_readBufferPtr+bufReadPos, _keyLength);
//compareRes = _readBinBuffer.CompareTo(key);
compareRes = HashBin.ArrayPtrCompare(_readBufferPtr + bufReadPos, ptrKey, compareLength);
if (compareRes == 0)
{
if (getDataAddr)
{
var foundLocation = FileStream.Position - lastRead + bufReadPos;
//FileStream.Position = ((((foundLocation) / _keyLength) << 4) + (long)_keyEnd + (long)HeaderLength);
AddrsFileStream.Position = (foundLocation / _keyLength) << 4;
dataAddr = new DataAddr
{
addr = _addrReader.ReadUInt64(),
len = _addrReader.ReadUInt64()
};
return(true);
}
dataAddr = default;
return(true);
}
if (compareRes == 1)
{
dataAddr = default;
return(false);
}
//
// if (HashBin.ArrayPtrEqualCompare(ptrKey, _readBufferPtr+bufReadPos, compareLength))
// {
// if (getDataAddr)
// {
// var foundLocation = FileStream.Position - lastRead + bufReadPos;
// FileStream.Position = ((((foundLocation) / _keyLength) << 4) + (long)_keyEnd + (long)HeaderLength);
// dataAddr = new DataAddr();
// dataAddr.addr = _reader.ReadUInt64();
// dataAddr.len = _reader.ReadUInt64();
// return true;
// }
//
// dataAddr = default;
// return true;
// }
bufReadPos += _keyLength;
}
}
}
dataAddr = default;
return(false);
}
public bool GetKeyDataAddr(HashBin key, out DataAddr dataAddr, KeyHint hint = default(KeyHint), byte skipBytes = 0)
{
return(HasKey(key, out dataAddr, true, hint, skipBytes));
}
public bool HasKey(HashBin key, KeyHint hint = default(KeyHint))
{
return(HasKey(key, out var unused, false, hint));
}
static void SortAndWrite(List <string> batchFiles, ulong keyCount, CacheyBashi cb, ushort keyLength, string outFile)
{
using var streams = new StreamCollection();
var batches = new List <CurrentBatchInfo>();
foreach (var batchFile in batchFiles)
{
var stream = new FileStream(batchFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);//File.OpenRead(batchFile);
streams.Streams.Add(stream);
batches.Add(new CurrentBatchInfo(keyLength, stream));
}
var cbKeyFileStream = cb.CbKey.FileStream;
var cbKeyWritter = new BinaryWriter(cbKeyFileStream);
cbKeyWritter.Write(keyCount);
var cbKeyAddrFileStream = cb.CbKey.AddrsFileStream;
var cbKeyAddrsWritter = new BinaryWriter(cbKeyAddrFileStream);
var remainingBatches = new List <CurrentBatchInfo>();
remainingBatches.AddRange(batches);
var currentKeyIndex = -1;
var currentKeyRangeStartAddr = cbKeyFileStream.Position;
ulong keysWritten = 0;
var addrOffset = cb.CbKey.HeaderLength + (keyCount * keyLength);
HashBin lastHash = null;
// #if DEBUG
// var debugHash = new HashBin("0000000000000000000000000000d8f4");
// #endif
foreach (var nextItem in SortAndDedupe(remainingBatches))
{
//update the index if we've reached the end of a key range
var keyIndex = cb.CbIndex.GetKeyIndexFromKey(nextItem.CurrentHashBin);
if (currentKeyIndex == -1)
{
currentKeyIndex = keyIndex;
}
else if (currentKeyIndex != keyIndex)//we've reached the end of a key range
{
var end = cbKeyFileStream.Position - keyLength;
cb.CbIndex.SetHintForKey(lastHash.Hash, new KeyHint()
{
StartAddr = (ulong)currentKeyRangeStartAddr,
EndAddr = (ulong)end
});
currentKeyRangeStartAddr = cbKeyFileStream.Position;
currentKeyIndex = keyIndex;
}
//write the key to the key file
cbKeyFileStream.Write(nextItem.CurrentHashBin.Hash, 0, nextItem.CurrentHashBin.Length);
//write the data addrs to the addr file
cbKeyAddrsWritter.Write(nextItem.CurrentAddr.addr);
cbKeyAddrsWritter.Write(nextItem.CurrentAddr.len);
//finally move to the next key in the batch
lastHash = nextItem.CurrentHashBin;
keysWritten++;
}
//don't forget to set the last item's key hint!
cb.CbIndex.SetHintForKey(lastHash.Hash, new KeyHint()
{
StartAddr = (ulong)currentKeyRangeStartAddr,
EndAddr = (ulong)cbKeyFileStream.Position - keyLength
});
//write the index out to disk
cb.CbIndex.WriteToDisk();
//tell cbKey to update stats
cb.CbKey.PostWriteUpdate();
//cleanup the batch files.
streams.Dispose();
foreach (var batchFile in batchFiles)
{
File.Delete(batchFile);
}
}