示例#1
0
        internal byte[] GetData(int blocknumber, List <int> usedblocks)
        {
            lock (_lock)
            {
                AllocationBlock ab = FillAllocationBlock(blocknumber);
                usedblocks = ab.Blocks;
                byte[] data   = new byte[ab.datalength];
                long   offset = 0;
                int    len    = ab.datalength;
                int    dbsize = _BlockSize - _blockheader.Length - ab.keylen;
                ab.Blocks.ForEach(x =>
                {
                    byte[] b = _datastore.ReadBlock(x);
                    int c    = len;
                    if (c > dbsize)
                    {
                        c = dbsize;
                    }
                    Buffer.BlockCopy(b, _blockheader.Length + ab.keylen, data, (int)offset, c);
                    offset += c;
                    len    -= c;
                });
                if (ab.isCompressed)
                {
                    data = MiniLZO.Decompress(data);
                }

                return(data);
            }
        }
示例#2
0
        private AllocationBlock FillAllocationBlock(int blocknumber)
        {
            AllocationBlock ab = new AllocationBlock();

            ab.blocknumber = blocknumber;
            ab.Blocks.Add(blocknumber);

            byte[] b = _datastore.ReadBlockBytes(blocknumber, _blockheader.Length + 255);

            int blocknumexpected = 0;

            int next = ParseBlockHeader(ab, b, blocknumexpected);

            blocknumexpected++;

            while (next > 0)
            {
                ab.Blocks.Add(next);
                b    = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen);
                next = ParseBlockHeader(ab, b, blocknumexpected);
                blocknumexpected++;
            }

            return(ab);
        }
示例#3
0
        private byte[] readblockdata(AllocationBlock ab)
        {
            byte[] data   = new byte[ab.datalength];
            long   offset = 0;
            int    len    = ab.datalength;
            int    dbsize = _BlockSize - _blockheader.Length - ab.keylen;

            ab.Blocks.ForEach(x =>
            {
                byte[] b = _datastore.ReadBlock(x);
                int c    = len;
                if (c > dbsize)
                {
                    c = dbsize;
                }
                Buffer.BlockCopy(b, _blockheader.Length + ab.keylen, data, (int)offset, c);
                offset += c;
                len    -= c;
            });
            if (ab.isCompressed)
            {
                data = MiniLZO.Decompress(data);
            }
            return(data);
        }
示例#4
0
        public bool SetObjectHF(string key, object obj)
        {
            byte[] k = Helper.GetBytes(key);
            if (k.Length > 255)
            {
                _log.Error("Key length > 255 : " + key);
                throw new Exception("Key must be less than 255 characters");
                //return false;
            }
            lock (_lock)
            {
                if (_isDirty == false)
                {
                    WriteDirtyFile();
                }

                AllocationBlock ab         = null;
                int             firstblock = 0;
                if (_keys.Get(key, out firstblock))// key exists already
                {
                    ab = FillAllocationBlock(firstblock);
                }

                SaveNew(key, k, obj);
                if (ab != null)
                {
                    // free old blocks
                    ab.Blocks.Add(ab.blocknumber);
                    _datastore.FreeBlocks(ab.Blocks);
                }
                return(true);
            }
        }
示例#5
0
        public int Increment(string key, int amount)
        {
            byte[] k = Helper.GetBytes(key);
            if (k.Length > 255)
            {
                _log.Error("Key length > 255 : " + key);
                throw new Exception("Key must be less than 255 characters");
                //return false;
            }
            lock (_lock)
            {
                if (_isDirty == false)
                {
                    WriteDirtyFile();
                }

                AllocationBlock ab         = null;
                int             firstblock = 0;
                if (_keys.Get(key, out firstblock))// key exists already
                {
                    ab = FillAllocationBlock(firstblock);
                }

                object obj = amount;
                if (ab.deleteKey == false)
                {
                    byte[] data = readblockdata(ab);

                    obj = fastBinaryJSON.BJSON.ToObject(data);

                    // add here
                    if (obj is int)
                    {
                        obj = ((int)obj) + amount;
                    }
                    else if (obj is long)
                    {
                        obj = ((long)obj) + amount;
                    }
                    else if (obj is decimal)
                    {
                        obj = ((decimal)obj) + amount;
                    }
                    else
                    {
                        return((int)obj);
                    }
                }

                SaveNew(key, k, obj);
                if (ab != null)
                {
                    // free old blocks
                    ab.Blocks.Add(ab.blocknumber);
                    _datastore.FreeBlocks(ab.Blocks);
                }
                return((int)obj);
            }
        }
示例#6
0
        internal byte[] GetData(int blocknumber, List <int> usedblocks)
        {
            lock (_lock)
            {
                AllocationBlock ab = FillAllocationBlock(blocknumber);
                usedblocks = ab.Blocks;
                byte[] data = readblockdata(ab);

                return(data);
            }
        }
示例#7
0
        internal int SaveData(string key, byte[] data)
        {
            lock (_lock)
            {
                byte[]          kb = Helper.GetBytes(key);
                AllocationBlock ab = new AllocationBlock();
                ab.key          = key;
                ab.keylen       = (byte)kb.Length;
                ab.isCompressed = false;
                ab.isBinaryJSON = true;
                ab.datalength   = data.Length;

                return(internalSave(kb, data, ab));
            }
        }
示例#8
0
        private int internalSave(byte[] keybytes, byte[] data, AllocationBlock ab)
        {
            ab.Blocks = new List <int>();
            int firstblock = _datastore.GetFreeBlockNumber();

            ab.Blocks.Add(firstblock);
            int blocknum = firstblock;

            byte[] header     = CreateAllocHeader(ab, keybytes);
            int    dblocksize = _BlockSize - header.Length;
            int    offset     = 0;
            // compute data block count
            int datablockcount = (data.Length / dblocksize) + 1;
            // save data blocks
            int counter = 0;
            int len     = data.Length;

            while (datablockcount > 0)
            {
                datablockcount--;
                int next = 0;
                if (datablockcount > 0)
                {
                    next = _datastore.GetFreeBlockNumber();
                }
                Buffer.BlockCopy(Helper.GetBytes(counter, false), 0, header, 0, 4); // set block number
                Buffer.BlockCopy(Helper.GetBytes(next, false), 0, header, 4, 4);    // set next pointer

                _datastore.SeekBlock(blocknum);
                _datastore.WriteBlockBytes(header, 0, header.Length);
                int c = len;
                if (c > dblocksize)
                {
                    c = dblocksize;
                }
                _datastore.WriteBlockBytes(data, offset, c);

                if (next > 0)
                {
                    blocknum = next;
                    ab.Blocks.Add(next);
                }
                offset += c;
                len    -= c;
                counter++;
            }
            return(firstblock);
        }
示例#9
0
        // for .string files
        internal int SaveData(string key, byte[] data, out List <int> blocks)
        {
            lock (_lock)
            {
                byte[]          kb = Helper.GetBytes(key);
                AllocationBlock ab = new AllocationBlock();
                ab.key          = key;
                ab.keylen       = (byte)kb.Length;
                ab.isCompressed = false;
                ab.isBinaryJSON = true;
                ab.datalength   = data.Length;

                int firstblock = internalSave(kb, data, ab);
                blocks = ab.Blocks;
                return(firstblock);
            }
        }
示例#10
0
        public object GetObjectHF(string key)
        {
            lock (_lock)
            {
                int alloc;
                if (_keys.Get(key, out alloc))
                {
                    AllocationBlock ab = FillAllocationBlock(alloc);
                    if (ab.deleteKey == false)
                    {
                        byte[] data = readblockdata(ab);

                        return(fastBinaryJSON.BJSON.ToObject(data));
                    }
                }
            }

            return(null);
        }
示例#11
0
        private int ParseBlockHeader(AllocationBlock ab, byte[] b, int blocknumberexpected)
        {
            int bnum = Helper.ToInt32(b, 0);

            if (bnum != blocknumberexpected)
            {
                _log.Error("Block numbers does not match, looking for : " + blocknumberexpected);
                //throw new Exception("Block numbers does not match, looking for : " + blocknumberexpected);
                return(-1);
            }
            if (b[14] != 1)
            {
                _log.Error("Expecting string keys only, got : " + b[14]);
                //throw new Exception("Expecting string keys only, got : " + b[11]);
                return(-1);
            }

            int next = Helper.ToInt32(b, 4);

            if (ab.keylen == 0)
            {
                byte flags = b[8];

                if ((flags & 0x01) > 0)
                {
                    ab.isCompressed = true;
                }
                if ((flags & 0x02) > 0)
                {
                    ab.isBinaryJSON = true;
                }
                if ((flags & 0x04) > 0)
                {
                    ab.deleteKey = true;
                }

                ab.datalength = Helper.ToInt32(b, 9);
                byte keylen = b[13];
                ab.keylen = keylen;
                ab.key    = Helper.GetString(b, _blockheader.Length, keylen);
            }
            return(next);
        }
示例#12
0
        public object GetObjectHF(string key)
        {
            lock (_lock)
            {
                int alloc;
                if (_keys.Get(key, out alloc))
                {
                    AllocationBlock ab = FillAllocationBlock(alloc);
                    if (ab.deleteKey == false)
                    {
                        byte[] data   = new byte[ab.datalength];
                        long   offset = 0;
                        int    len    = ab.datalength;
                        int    dbsize = _BlockSize - _blockheader.Length - ab.keylen;
                        ab.Blocks.ForEach(x =>
                        {
                            byte[] b = _datastore.ReadBlock(x);
                            int c    = len;
                            if (c > dbsize)
                            {
                                c = dbsize;
                            }
                            Buffer.BlockCopy(b, _blockheader.Length + ab.keylen, data, (int)offset, c);
                            offset += c;
                            len    -= c;
                        });
                        if (ab.isCompressed)
                        {
                            data = MiniLZO.Decompress(data);
                        }

                        return(fastBinaryJSON.BJSON.ToObject(data));
                    }
                }
            }

            return(null);
        }
示例#13
0
        private void SaveNew(string key, byte[] keybytes, object obj)
        {
            byte[]          data;
            AllocationBlock ab = new AllocationBlock();

            ab.key    = key;
            ab.keylen = (byte)keybytes.Length;

            data            = fastBinaryJSON.BJSON.ToBJSON(obj);
            ab.isBinaryJSON = true;

            if (data.Length > (int)Global.CompressDocumentOverKiloBytes * _KILOBYTE)
            {
                ab.isCompressed = true;
                data            = MiniLZO.Compress(data);
            }
            ab.datalength = data.Length;

            int firstblock = internalSave(keybytes, data, ab);

            // save keys
            _keys.Set(key, firstblock);
        }
示例#14
0
        private byte[] CreateAllocHeader(AllocationBlock ab, byte[] keybytes)
        {
            byte[] alloc = new byte[_blockheader.Length + keybytes.Length];

            if (ab.isCompressed)
            {
                alloc[8] = 1;
            }
            if (ab.isBinaryJSON)
            {
                alloc[8] += 2;
            }
            if (ab.deleteKey)
            {
                alloc[8] += 4;
            }

            Buffer.BlockCopy(Helper.GetBytes(ab.datalength, false), 0, alloc, 9, 4);
            alloc[13] = ab.keylen;
            alloc[14] = 1; // string keys for now
            Buffer.BlockCopy(keybytes, 0, alloc, _blockheader.Length, ab.keylen);

            return(alloc);
        }
示例#15
0
        public bool DeleteKeyHF(string key)
        {
            lock (_lock)
            {
                int alloc;
                if (_keys.Get(key, out alloc))
                {
                    if (_isDirty == false)
                    {
                        WriteDirtyFile();
                    }

                    byte[]          keybytes = Helper.GetBytes(key);
                    AllocationBlock ab       = FillAllocationBlock(alloc);

                    ab.keylen = (byte)keybytes.Length;

                    _keys.RemoveKey(key);// remove key from index

                    // write ab
                    ab.deleteKey  = true;
                    ab.datalength = 0;

                    byte[] header = CreateAllocHeader(ab, keybytes);

                    _datastore.SeekBlock(ab.blocknumber);
                    _datastore.WriteBlockBytes(header, 0, header.Length);

                    // free old data blocks
                    _datastore.FreeBlocks(ab.Blocks);

                    return(true);
                }
            }
            return(false);
        }
示例#16
0
 private byte[] readblockdata(AllocationBlock ab)
 {
     byte[] data = new byte[ab.datalength];
     long offset = 0;
     int len = ab.datalength;
     int dbsize = _BlockSize - _blockheader.Length - ab.keylen;
     ab.Blocks.ForEach(x =>
     {
         byte[] b = _datastore.ReadBlock(x);
         int c = len;
         if (c > dbsize) c = dbsize;
         Buffer.BlockCopy(b, _blockheader.Length + ab.keylen, data, (int)offset, c);
         offset += c;
         len -= c;
     });
     if (ab.isCompressed)
         data = MiniLZO.Decompress(data);
     return data;
 }
示例#17
0
        private void SaveNew(string key, byte[] keybytes, object obj)
        {
            byte[] data;
            AllocationBlock ab = new AllocationBlock();
            ab.key = key;
            ab.keylen = (byte)keybytes.Length;

            data = fastBinaryJSON.BJSON.ToBJSON(obj);
            ab.isBinaryJSON = true;

            if (data.Length > (int)Global.CompressDocumentOverKiloBytes * _KILOBYTE)
            {
                ab.isCompressed = true;
                data = MiniLZO.Compress(data);
            }
            ab.datalength = data.Length;

            int firstblock = internalSave(keybytes, data, ab);

            // save keys
            _keys.Set(key, firstblock);
        }
示例#18
0
        private void RebuildDataFiles()
        {
            MGIndex <string> keys = null;

            try
            {
                // remove old free list
                if (File.Exists(_Path + "data.bmp"))
                {
                    File.Delete(_Path + "data.bmp");
                }

                _datastore = new StorageFileHF(_Path + "data.mghf", Global.HighFrequencyKVDiskBlockSize);
                _BlockSize = _datastore.GetBlockSize();
                if (File.Exists(_Path + "keys.idx"))
                {
                    _log.Debug("removing old keys index");
                    foreach (var f in Directory.GetFiles(_Path, "keys.*"))
                    {
                        File.Delete(f);
                    }
                }

                keys = new MGIndex <string>(_Path, "keys.idx", 255, /*Global.PageItemCount,*/ false);

                WAHBitArray visited = new WAHBitArray();

                int c = _datastore.NumberofBlocks();

                for (int i = 0; i < c; i++) // go through blocks
                {
                    if (visited.Get(i))
                    {
                        continue;
                    }
                    byte[] b    = _datastore.ReadBlockBytes(i, _blockheader.Length + 255);
                    int    bnum = Helper.ToInt32(b, 0);
                    if (bnum > 0) // check if a start block
                    {
                        visited.Set(i, true);
                        _datastore.FreeBlock(i); // mark as free
                        continue;
                    }

                    AllocationBlock ab = new AllocationBlock();
                    // start block found
                    int blocknumexpected = 0;

                    int             next     = ParseBlockHeader(ab, b, blocknumexpected);
                    int             last     = 0;
                    bool            freelast = false;
                    AllocationBlock old      = null;

                    if (keys.Get(ab.key, out last))
                    {
                        old      = this.FillAllocationBlock(last);
                        freelast = true;
                    }
                    blocknumexpected++;
                    bool failed = false;
                    if (ab.deleteKey == false)
                    {
                        while (next > 0) // read the blocks
                        {
                            ab.Blocks.Add(next);
                            b    = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen);
                            next = ParseBlockHeader(ab, b, blocknumexpected);
                            if (next == -1) // non matching block
                            {
                                failed = true;
                                break;
                            }
                            blocknumexpected++;
                        }
                    }
                    else
                    {
                        failed = true;
                        keys.RemoveKey(ab.key);
                    }
                    // new data ok
                    if (failed == false)
                    {
                        keys.Set(ab.key, ab.blocknumber); // valid block found
                        if (freelast)                     // free the old blocks
                        {
                            _datastore.FreeBlocks(old.Blocks);
                        }
                    }

                    visited.Set(i, true);
                }

                // all ok delete temp.$ file
                if (File.Exists(_Path + _dirtyFilename))
                {
                    File.Delete(_Path + _dirtyFilename);
                }
            }
            catch (Exception ex)
            {
                _log.Error(ex);
            }
            finally
            {
                _log.Debug("Shutting down files and index");
                _datastore.Shutdown();
                keys.SaveIndex();
                keys.Shutdown();
            }
        }
示例#19
0
        private int internalSave(byte[] keybytes, byte[] data, AllocationBlock ab)
        {
            int firstblock = _datastore.GetFreeBlockNumber();
            int blocknum = firstblock;
            byte[] header = CreateAllocHeader(ab, keybytes);
            int dblocksize = _BlockSize - header.Length;
            int offset = 0;
            // compute data block count
            int datablockcount = (data.Length / dblocksize) + 1;
            // save data blocks
            int counter = 0;
            int len = data.Length;
            while (datablockcount > 0)
            {
                datablockcount--;
                int next = 0;
                if (datablockcount > 0)
                    next = _datastore.GetFreeBlockNumber();

                Buffer.BlockCopy(Helper.GetBytes(counter, false), 0, header, 0, 4);    // set block number
                Buffer.BlockCopy(Helper.GetBytes(next, false), 0, header, 4, 4); // set next pointer

                _datastore.SeekBlock(blocknum);
                _datastore.WriteBlockBytes(header, 0, header.Length);
                int c = len;
                if (c > dblocksize)
                    c = dblocksize;
                _datastore.WriteBlockBytes(data, offset, c);

                if (next > 0)
                    blocknum = next;
                offset += c;
                len -= c;
                counter++;
            }
            return firstblock;
        }
示例#20
0
        private byte[] CreateAllocHeader(AllocationBlock ab, byte[] keybytes)
        {
            byte[] alloc = new byte[_blockheader.Length + keybytes.Length];

            if (ab.isCompressed)
                alloc[8] = 1;
            if (ab.isBinaryJSON)
                alloc[8] += 2;
            if (ab.deleteKey)
                alloc[8] += 4;

            Buffer.BlockCopy(Helper.GetBytes(ab.datalength, false), 0, alloc, 9, 4);
            alloc[13] = ab.keylen;
            alloc[14] = 1; // string keys for now
            Buffer.BlockCopy(keybytes, 0, alloc, _blockheader.Length, ab.keylen);

            return alloc;
        }
示例#21
0
        internal int SaveData(string key, byte[] data)
        {
            lock (_lock)
            {
                byte[] kb = Helper.GetBytes(key);
                AllocationBlock ab = new AllocationBlock();
                ab.key = key;
                ab.keylen = (byte)kb.Length;
                ab.isCompressed = false;
                ab.isBinaryJSON = true;
                ab.datalength = data.Length;

                return internalSave(kb, data, ab);
            }
        }
示例#22
0
        private void RebuildDataFiles()
        {
            MGIndex<string> keys = null;
            try
            {
                // remove old free list
                if (File.Exists(_Path + "data.bmp"))
                    File.Delete(_Path + "data.bmp");

                _datastore = new StorageFileHF(_Path + "data.mghf", Global.HighFrequencyKVDiskBlockSize);
                _BlockSize = _datastore.GetBlockSize();
                if (File.Exists(_Path + "keys.idx"))
                {
                    _log.Debug("removing old keys index");
                    foreach (var f in Directory.GetFiles(_Path, "keys.*"))
                        File.Delete(f);
                }

                keys = new MGIndex<string>(_Path, "keys.idx", 255, Global.PageItemCount, false);

                WAHBitArray visited = new WAHBitArray();

                int c = _datastore.NumberofBlocks();

                for (int i = 0; i < c; i++) // go through blocks
                {
                    if (visited.Get(i))
                        continue;
                    byte[] b = _datastore.ReadBlockBytes(i, _blockheader.Length + 255);
                    int bnum = Helper.ToInt32(b, 0);
                    if (bnum > 0) // check if a start block
                    {
                        visited.Set(i, true);
                        _datastore.FreeBlock(i); // mark as free
                        continue;
                    }

                    AllocationBlock ab = new AllocationBlock();
                    // start block found
                    int blocknumexpected = 0;

                    int next = ParseBlockHeader(ab, b, blocknumexpected);
                    int last = 0;
                    bool freelast = false;
                    AllocationBlock old = null;

                    if (keys.Get(ab.key, out last))
                    {
                        old = this.FillAllocationBlock(last);
                        freelast = true;
                    }
                    blocknumexpected++;
                    bool failed = false;
                    if (ab.deleteKey == false)
                    {
                        while (next > 0) // read the blocks
                        {
                            ab.Blocks.Add(next);
                            b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen);
                            next = ParseBlockHeader(ab, b, blocknumexpected);
                            if (next == -1) // non matching block
                            {
                                failed = true;
                                break;
                            }
                            blocknumexpected++;
                        }
                    }
                    else
                    {
                        failed = true;
                        keys.RemoveKey(ab.key);
                    }
                    // new data ok
                    if (failed == false)
                    {
                        keys.Set(ab.key, ab.blocknumber);// valid block found
                        if (freelast)// free the old blocks
                            _datastore.FreeBlocks(old.Blocks);
                    }

                    visited.Set(i, true);
                }

                // all ok delete temp.$ file
                if (File.Exists(_Path + _dirtyFilename))
                    File.Delete(_Path + _dirtyFilename);
            }
            catch (Exception ex)
            {
                _log.Error(ex);
            }
            finally
            {
                _log.Debug("Shutting down files and index");
                _datastore.Shutdown();
                keys.SaveIndex();
                keys.Shutdown();
            }
        }
示例#23
0
        private int ParseBlockHeader(AllocationBlock ab, byte[] b, int blocknumberexpected)
        {
            int bnum = Helper.ToInt32(b, 0);
            if (bnum != blocknumberexpected)
            {
                _log.Error("Block numbers does not match, looking for : " + blocknumberexpected);
                //throw new Exception("Block numbers does not match, looking for : " + blocknumberexpected);
                return -1;
            }
            if (b[14] != 1)
            {
                _log.Error("Expecting string keys only, got : " + b[14]);
                //throw new Exception("Expecting string keys only, got : " + b[11]);
                return -1;
            }

            int next = Helper.ToInt32(b, 4);

            if (ab.keylen == 0)
            {
                byte flags = b[8];

                if ((flags & 0x01) > 0)
                    ab.isCompressed = true;
                if ((flags & 0x02) > 0)
                    ab.isBinaryJSON = true;
                if ((flags & 0x04) > 0)
                    ab.deleteKey = true;

                ab.datalength = Helper.ToInt32(b, 9);
                byte keylen = b[13];
                ab.keylen = keylen;
                ab.key = Helper.GetString(b, _blockheader.Length, keylen);
            }
            return next;
        }
示例#24
0
        private AllocationBlock FillAllocationBlock(int blocknumber)
        {
            AllocationBlock ab = new AllocationBlock();

            ab.blocknumber = blocknumber;
            ab.Blocks.Add(blocknumber);

            byte[] b = _datastore.ReadBlockBytes(blocknumber, _blockheader.Length + 255);

            int blocknumexpected = 0;

            int next = ParseBlockHeader(ab, b, blocknumexpected);

            blocknumexpected++;

            while (next > 0)
            {
                ab.Blocks.Add(next);
                b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen);
                next = ParseBlockHeader(ab, b, blocknumexpected);
                blocknumexpected++;
            }

            return ab;
        }