Ejemplo n.º 1
0
        public CbKeyFixed(string keyFile, ushort keyLength = 16, bool createNew = false)
        {
            _ulongSize   = sizeof(ulong);
            HeaderLength = _ulongSize;
            _keyStart    = HeaderLength;

            _keyLength = keyLength;
            _file      = keyFile;
            if (File.Exists(_file) && createNew)
            {
                File.Delete(_file);
            }

            if (File.Exists(addrFile) && createNew)
            {
                File.Delete(addrFile);
            }

            FileStream        = File.Open(_file, FileMode.OpenOrCreate);
            AddrsFileStream   = File.Open(addrFile, FileMode.OpenOrCreate);
            _reader           = new BinaryReader(FileStream);
            _readBuffer       = new byte[_keyLength * 10000];
            _readBufferHandle = GCHandle.Alloc(_readBuffer, GCHandleType.Pinned);
            _readBufferPtr    = (byte *)_readBufferHandle.AddrOfPinnedObject();
            _readBinBuffer    = new HashBin(new byte[_keyLength], false);
            _readBinBuffer.SetFromPartialArray(_readBuffer, 0, _keyLength, false);

            _addrReader = new BinaryReader(AddrsFileStream);

            if (FileStream.Length > 8)
            {
                _count  = _reader.ReadUInt64();
                _keyEnd = _count * _keyLength;
            }
        }
Ejemplo n.º 2
0
        public byte[] GetValue(HashBin key)
        {
            var  hint  = CbIndex.GetAddressHintForKey(key);
            bool found = CbKey.GetKeyDataAddr(key, out var addr, hint, CbIndex.IndexKeyLen);

            if (found)
            {
                return(CbData.GetValue(addr));
            }
            return(null);
        }
Ejemplo n.º 3
0
        public bool HasKey(HashBin key)
        {
            var hint = CbIndex.GetAddressHintForKey(key);

            return(CbKey.HasKey(key, out var unused, false, hint, CbIndex.IndexKeyLen));
        }
Ejemplo n.º 4
0
 public KeyHint GetAddressHintForKey(HashBin key)
 {
     return(GetAddressHintForKey(key.Hash));
 }
Ejemplo n.º 5
0
 public int GetKeyIndexFromKey(HashBin key)
 {
     return(GetKeyIndexFromKey(key.Hash));
 }
Ejemplo n.º 6
0
        public bool HasKey(HashBin key, out DataAddr dataAddr, bool getDataAddr = false, KeyHint hint = default(KeyHint), byte skipBytes = 0)
        {
            if (key.Length != _keyLength)
            {
                throw new ArgumentException($"Key must be {_keyLength} bytes");
            }

            // bool hasHint = hint.StartAddr >= _keyStart &&
            //                hint.EndAddr >= hint.StartAddr
            //                && hint.EndAddr <= _keyEnd;

            int compareLength = _keyLength - skipBytes;

            if (!(hint.StartAddr >= _keyStart &&
                  hint.EndAddr >= hint.StartAddr &&
                  hint.EndAddr <= _keyEnd))
            {
                hint.StartAddr = _keyStart;
                hint.EndAddr   = _keyEnd;
                compareLength  = _keyLength;
            }

            FileStream.Position = (long)hint.StartAddr;
            var  remaining = (long)(hint.EndAddr - hint.StartAddr + _keyLength);
            long lastRead;
            int  amountToRead;
            var  bufReadPos = 0;
            int  compareRes;

            fixed(byte *ptrKey = &key.Hash[0])
            {
                while (remaining > 0)
                {
                    amountToRead = (int)(remaining < _readBuffer.Length ? remaining : _readBuffer.Length);
                    lastRead     = FileStream.Read(_readBuffer, 0, amountToRead);
                    remaining   -= lastRead;
                    bufReadPos   = 0;
                    //loop until buffer read
                    while (bufReadPos < lastRead)
                    {
                        //_readBinBuffer.SetPointer(_readBufferPtr+bufReadPos, _keyLength);
                        //compareRes = _readBinBuffer.CompareTo(key);
                        compareRes = HashBin.ArrayPtrCompare(_readBufferPtr + bufReadPos, ptrKey, compareLength);

                        if (compareRes == 0)
                        {
                            if (getDataAddr)
                            {
                                var foundLocation = FileStream.Position - lastRead + bufReadPos;
                                //FileStream.Position = ((((foundLocation) / _keyLength) << 4) + (long)_keyEnd + (long)HeaderLength);
                                AddrsFileStream.Position = (foundLocation / _keyLength) << 4;
                                dataAddr = new DataAddr
                                {
                                    addr = _addrReader.ReadUInt64(),
                                    len  = _addrReader.ReadUInt64()
                                };
                                return(true);
                            }

                            dataAddr = default;
                            return(true);
                        }

                        if (compareRes == 1)
                        {
                            dataAddr = default;
                            return(false);
                        }
                        //
                        // if (HashBin.ArrayPtrEqualCompare(ptrKey, _readBufferPtr+bufReadPos, compareLength))
                        // {
                        //     if (getDataAddr)
                        //     {
                        //         var foundLocation = FileStream.Position - lastRead + bufReadPos;
                        //         FileStream.Position = ((((foundLocation) / _keyLength) << 4) + (long)_keyEnd + (long)HeaderLength);
                        //         dataAddr = new DataAddr();
                        //         dataAddr.addr = _reader.ReadUInt64();
                        //         dataAddr.len = _reader.ReadUInt64();
                        //         return true;
                        //     }
                        //
                        //     dataAddr = default;
                        //     return true;
                        // }



                        bufReadPos += _keyLength;
                    }
                }
            }

            dataAddr = default;
            return(false);
        }
Ejemplo n.º 7
0
 public bool GetKeyDataAddr(HashBin key, out DataAddr dataAddr, KeyHint hint = default(KeyHint), byte skipBytes = 0)
 {
     return(HasKey(key, out dataAddr, true, hint, skipBytes));
 }
Ejemplo n.º 8
0
 public bool HasKey(HashBin key, KeyHint hint = default(KeyHint))
 {
     return(HasKey(key, out var unused, false, hint));
 }
Ejemplo n.º 9
0
        static void SortAndWrite(List <string> batchFiles, ulong keyCount, CacheyBashi cb, ushort keyLength, string outFile)
        {
            using var streams = new StreamCollection();
            var batches = new List <CurrentBatchInfo>();

            foreach (var batchFile in batchFiles)
            {
                var stream = new FileStream(batchFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);//File.OpenRead(batchFile);
                streams.Streams.Add(stream);
                batches.Add(new CurrentBatchInfo(keyLength, stream));
            }

            var cbKeyFileStream = cb.CbKey.FileStream;
            var cbKeyWritter    = new BinaryWriter(cbKeyFileStream);

            cbKeyWritter.Write(keyCount);

            var cbKeyAddrFileStream = cb.CbKey.AddrsFileStream;
            var cbKeyAddrsWritter   = new BinaryWriter(cbKeyAddrFileStream);

            var remainingBatches = new List <CurrentBatchInfo>();

            remainingBatches.AddRange(batches);

            var currentKeyIndex          = -1;
            var currentKeyRangeStartAddr = cbKeyFileStream.Position;

            ulong   keysWritten = 0;
            var     addrOffset  = cb.CbKey.HeaderLength + (keyCount * keyLength);
            HashBin lastHash    = null;

// #if DEBUG
//             var debugHash = new HashBin("0000000000000000000000000000d8f4");
// #endif

            foreach (var nextItem in SortAndDedupe(remainingBatches))
            {
                //update the index if we've reached the end of a key range
                var keyIndex = cb.CbIndex.GetKeyIndexFromKey(nextItem.CurrentHashBin);
                if (currentKeyIndex == -1)
                {
                    currentKeyIndex = keyIndex;
                }
                else if (currentKeyIndex != keyIndex)//we've reached the end of a key range
                {
                    var end = cbKeyFileStream.Position - keyLength;
                    cb.CbIndex.SetHintForKey(lastHash.Hash, new KeyHint()
                    {
                        StartAddr = (ulong)currentKeyRangeStartAddr,
                        EndAddr   = (ulong)end
                    });
                    currentKeyRangeStartAddr = cbKeyFileStream.Position;
                    currentKeyIndex          = keyIndex;
                }

                //write the key to the key  file
                cbKeyFileStream.Write(nextItem.CurrentHashBin.Hash, 0, nextItem.CurrentHashBin.Length);

                //write the data addrs to the addr file
                cbKeyAddrsWritter.Write(nextItem.CurrentAddr.addr);
                cbKeyAddrsWritter.Write(nextItem.CurrentAddr.len);

                //finally move to the next key in the batch
                lastHash = nextItem.CurrentHashBin;

                keysWritten++;
            }

            //don't forget to set the last item's key hint!
            cb.CbIndex.SetHintForKey(lastHash.Hash, new KeyHint()
            {
                StartAddr = (ulong)currentKeyRangeStartAddr,
                EndAddr   = (ulong)cbKeyFileStream.Position - keyLength
            });

            //write the index out to disk
            cb.CbIndex.WriteToDisk();

            //tell cbKey to update stats
            cb.CbKey.PostWriteUpdate();

            //cleanup the batch files.
            streams.Dispose();
            foreach (var batchFile in batchFiles)
            {
                File.Delete(batchFile);
            }
        }