private void CreatePageListData(SafeSortedList <T, PageInfo> _pages, int offset, int rowindex, int counter, byte[] page, int blocknum) { int idx = rowindex + _rowSize * counter; // key bytes byte[] kk; byte size; if (_externalStrings == false) { kk = _T.GetBytes(_pages.GetKey(counter + offset)); size = (byte)kk.Length; if (size > _maxKeySize) { size = _maxKeySize; } } else { kk = new byte[4]; Buffer.BlockCopy(Helper.GetBytes(counter + offset, false), 0, kk, 0, 4); size = 4; } // key size = 1 byte page[idx] = size; Buffer.BlockCopy(kk, 0, page, idx + 1, page[idx]); // offset = 4 bytes byte[] b = Helper.GetBytes(_pages.GetValue(offset + counter).PageNumber, false); Buffer.BlockCopy(b, 0, page, idx + 1 + _maxKeySize, b.Length); // add counts b = Helper.GetBytes(_pages.GetValue(offset + counter).UniqueCount, false); Buffer.BlockCopy(b, 0, page, idx + 1 + _maxKeySize + 4, b.Length); // FEATURE : add dup counts }
public void Commit(bool freeMemory) { if (_isDirty == false) { return; } using (new L(this)) { log.Debug("writing " + _FileName); int[] keys = _cache.Keys(); Array.Sort(keys); foreach (int k in keys) { WAHBitArray bmp = null; if (_cache.TryGetValue(k, out bmp) && bmp.isDirty) { SaveBitmap(k, bmp); bmp.FreeMemory(); bmp.isDirty = false; } } Flush(); if (freeMemory) { _cache = //new SafeDictionary<int, WAHBitArray>(); new SafeSortedList <int, WAHBitArray>(); log.Debug(" freeing cache"); } _isDirty = false; } }
public WAHBitArray(TYPE type, uint[] ints) { _state = type; switch (type) { case TYPE.WAH: _compressed = ints; Uncompress(); _state = TYPE.Bitarray; _compressed = null; break; case TYPE.Bitarray: _uncompressed = ints; break; case TYPE.Indexes: if (Global.UseLessMemoryStructures) { _offsets = new SafeSortedList <uint, bool>(); } else { _offsets = new SafeDictionary <uint, bool>(); } //new Dictionary<uint, bool>(); foreach (var i in ints) { _offsets.Add(i, true); } break; } }
public static MGRB Fill(long count) { if (count == 0) { return(new MGRB()); } var con = new SafeSortedList <int, Container>(); int i = 0; long c = count; while (count > 0) { if (count > Container.BSize) { con.Add(i, new BitmapContainer(true)); } else { con.Add(i, new BitmapContainer((int)count)); } count -= Container.BSize; i++; } return(new MGRB(con, c)); }
private void LoadWords() { lock (_lock) { if (_words == null) { _words = new SafeSortedList <string, int>(); } if (File.Exists(_Path + _FileName + ".words") == false) { return; } // load words byte[] b = File.ReadAllBytes(_Path + _FileName + ".words"); if (b.Length == 0) { return; } MemoryStream ms = new MemoryStream(b); BinaryReader br = new BinaryReader(ms, Encoding.UTF8); string s = br.ReadString(); while (s != "") { int off = br.ReadInt32(); _words.Add(s, off); try { s = br.ReadString(); } catch { s = ""; } } _log.Debug("Word Count = " + _words.Count); _loaded = true; } }
public MGRB And(MGRB B) { var v = new SafeSortedList <int, Container>(); var len = _size; if (B.Length < len) { len = B.Length; } var a = LastContainerIdx(); var b = B.LastContainerIdx(); var min = a; if (b < min) { min = b; } min++; for (int i = 0; i < min; i++) { Container ca = null; Container cb = null; _containers.TryGetValue(i, out ca); B._containers.TryGetValue(i, out cb); if (ca != null && cb != null) { v.Add(i, containerAND(ca, cb)); } } return(new MGRB(v, len)); }
public MGRB Not() { var con = new SafeSortedList <int, Container>(); foreach (var c in _containers) { con.Add(c.Key, c.Value.Not()); } return(new MGRB(con, _size)); }
internal MGRB(SafeSortedList <int, Container> containers) { _containers = containers; var k = _containers.Keys(); var l = k.Length - 1; if (l >= 0) { _size = (k[l] << 16) + _containers.GetValue(l).Size; } }
internal void SavePageList(SafeSortedList <T, PageInfo> _pages, List <int> diskpages) { lock (_fileLock) { T[] keys = _pages.Keys(); int blocknum = 0; // TODO : needed?? //if (_externalStrings) //{ // if (_pagelistalllocblock != null) // _strings.FreeBlocks(_pagelistalllocblock); // blocknum = _strings.SaveData("pagelist", BJSON.ToBJSON(keys, // new BJSONParameters { UseUnicodeStrings = false, UseTypedArrays = false })); // File.WriteAllBytes(_FileName + ".pagelist", Helper.GetBytes(blocknum, false)); //} // save page list int c = (_pages.Count() / Global.PageItemCount) + 1; // allocate pages needed while (c > diskpages.Count) { diskpages.Add(GetNewPageNumber()); } byte[] page = new byte[_PageLength]; for (int i = 0; i < (diskpages.Count - 1); i++) { byte[] block = CreateBlockHeader(1, Global.PageItemCount, diskpages[i + 1]); Buffer.BlockCopy(block, 0, page, 0, block.Length); for (int j = 0; j < Global.PageItemCount; j++) { CreatePageListData(_pages, i * Global.PageItemCount, block.Length, j, page, blocknum); } SeekPage(diskpages[i]); _file.Write(page, 0, page.Length); } c = _pages.Count() % Global.PageItemCount; byte[] lastblock = CreateBlockHeader(1, (ushort)c, -1); Buffer.BlockCopy(lastblock, 0, page, 0, lastblock.Length); int lastoffset = (_pages.Count() / Global.PageItemCount) * Global.PageItemCount; for (int j = 0; j < c; j++) { CreatePageListData(_pages, lastoffset, lastblock.Length, j, page, blocknum); } SeekPage(diskpages[diskpages.Count - 1]); _file.Write(page, 0, page.Length); } }
public WAHBitArray() { _state = TYPE.Indexes; if (Global.UseLessMemoryStructures) { _offsets = new SafeSortedList <uint, bool>(); } else { _offsets = new SafeDictionary <uint, bool>(); } }
internal void SavePageList(SafeSortedList <T, PageInfo> _pages, List <int> diskpages) { lock (_fileLock) { // save page list int c = (_pages.Count / Global.PageItemCount) + 1; // allocate pages needed while (c > diskpages.Count) { diskpages.Add(GetNewPageNumber()); } byte[] page = new byte[_PageLength]; for (int i = 0; i < (diskpages.Count - 1); i++) { byte[] block = CreateBlockHeader(1, Global.PageItemCount, diskpages[i + 1]); Buffer.BlockCopy(block, 0, page, 0, block.Length); for (int j = 0; j < Global.PageItemCount; j++) { CreatePageListData(_pages, i * Global.PageItemCount, block.Length, j, page); } SeekPage(diskpages[i]); _file.Write(page, 0, page.Length); } c = _pages.Count % Global.PageItemCount; byte[] lastblock = CreateBlockHeader(1, (ushort)c, -1); Buffer.BlockCopy(lastblock, 0, page, 0, lastblock.Length); int lastoffset = (_pages.Count / Global.PageItemCount) * Global.PageItemCount; for (int j = 0; j < c; j++) { CreatePageListData(_pages, lastoffset, lastblock.Length, j, page); } // [BRADS] This is null when shutting down sometimes if (_file == null) { return; } SeekPage(diskpages[diskpages.Count - 1]); _file.Write(page, 0, page.Length); } }
internal MGRB(SafeSortedList <int, Container> containers, long size) { _containers = containers; var k = _containers.Keys(); _size = size; if (size <= 0)//== -1) { _size = 0; var l = k.Length - 1; if (l >= 0) { _size = (k[l] << 16) + _containers.GetValue(l).Size; } } }
public void GetPageList(List <int> PageListDiskPages, SafeSortedList <T, PageInfo> PageList, out int lastIndexedRow) { lastIndexedRow = Helper.ToInt32(_FileHeader, 11); // load page list PageListDiskPages.Add(0); // first page list int nextpage = LoadPageListData(0, PageList); while (nextpage != -1) { nextpage = LoadPageListData(nextpage, PageList); if (nextpage != -1) { PageListDiskPages.Add(nextpage); } } }
public static MGRB Fill(long count) { if (count == 0) { return(new MGRB()); } var con = new SafeSortedList <int, Container>(); var c = count >> 16; for (int i = 0; i <= c; i++) { con.Add(i, new BitmapContainer(true)); } return(new MGRB(con)); }
public MGRB Or(MGRB B) { var v = new SafeSortedList <int, Container>(); var len = _size; if (B.Length > len) { len = B.Length; } var a = LastContainerIdx(); var b = B.LastContainerIdx(); var max = a; if (b > max) { max = b; } max++; for (int i = 0; i < max; i++) { Container ca = null; Container cb = null; _containers.TryGetValue(i, out ca); B._containers.TryGetValue(i, out cb); if (ca == null && cb != null) { v.Add(i, cb.Copy()); } else if (cb == null && ca != null) { v.Add(i, ca.Copy()); } else if (ca != null && cb != null) { v.Add(i, containerOR(ca, cb)); } } return(new MGRB(v, len)); }
public BitmapIndex(string path, string filename) { if (Global.UseLessMemoryStructures) { _cache = new SafeSortedList <int, WAHBitArray>(); } else { _cache = new SafeDictionary <int, WAHBitArray>(); } _FileName = Path.GetFileNameWithoutExtension(filename); _Path = path; if (_Path.EndsWith(Path.DirectorySeparatorChar.ToString()) == false) { _Path += Path.DirectorySeparatorChar.ToString(); } Initialize(); }
public MGRB Not(long count) { var con = new SafeSortedList <int, Container>(); var c = count >> 16; for (int i = 0; i <= c; i++) { Container a = null; _containers.TryGetValue(i, out a); if (a == null) { con.Add(i, new BitmapContainer(true)); } else { con.Add(i, a.Not()); } } return(new MGRB(con, count)); }
private int LoadPageListData(int page, SafeSortedList <T, PageInfo> PageList) { lock (_fileLock) { // load page list data int nextpage = -1; SeekPage(page); byte[] b = new byte[_PageLength]; _file.Read(b, 0, _PageLength); if (b[0] == _BlockHeader[0] && b[1] == _BlockHeader[1] && b[2] == _BlockHeader[2] && b[3] == _BlockHeader[3]) { short count = Helper.ToInt16(b, 5); if (count > _PageNodeCount) { throw new Exception("Count > node size"); } nextpage = Helper.ToInt32(b, 11); int index = _BlockHeader.Length; for (int i = 0; i < count; i++) { int idx = index + _rowSize * i; byte ks = b[idx]; T key = _T.GetObject(b, idx + 1, ks); int pagenum = Helper.ToInt32(b, idx + 1 + _maxKeySize); // add counts int unique = Helper.ToInt32(b, idx + 1 + _maxKeySize + 4); // FEATURE : add dup count PageList.Add(key, new PageInfo(pagenum, unique, 0)); } } else { throw new Exception("Page List header is invalid"); } return(nextpage); } }
private void CreatePageListData(SafeSortedList <T, PageInfo> _pages, int i, byte[] page, int index, int j) { int idx = index + _rowSize * j; // key bytes byte[] kk = _T.GetBytes(_pages.GetKey(j + i)); byte size = (byte)kk.Length; if (size > _maxKeySize) { size = _maxKeySize; } // key size = 1 byte page[idx] = size; Buffer.BlockCopy(kk, 0, page, idx + 1, page[idx]); // offset = 4 bytes byte[] b = Helper.GetBytes(_pages.GetValue(i + j).PageNumber, false); Buffer.BlockCopy(b, 0, page, idx + 1 + _maxKeySize, b.Length); // add counts b = Helper.GetBytes(_pages.GetValue(i + j).UniqueCount, false); Buffer.BlockCopy(b, 0, page, idx + 1 + _maxKeySize + 4, b.Length); // FEATURE : add dup counts }
internal MGRB(SafeSortedList <int, Container> containers) : this(containers, -1) { }
private int LoadPageListData(int page, SafeSortedList <T, PageInfo> PageList) { lock (_fileLock) { // load page list data int nextpage = -1; SeekPage(page); byte[] b = new byte[_PageLength]; _file.Read(b, 0, _PageLength); if (b[0] == _BlockHeader[0] && b[1] == _BlockHeader[1] && b[2] == _BlockHeader[2] && b[3] == _BlockHeader[3]) { short count = Helper.ToInt16(b, 5); if (count > _PageNodeCount) { throw new Exception("Count > node size"); } nextpage = Helper.ToInt32(b, 11); int index = _BlockHeader.Length; object[] keys = null; // TODO : needed?? //if (File.Exists(_FileName + ".pagelist")) //{ // var bn = File.ReadAllBytes(_FileName + ".pagelist"); // int blknum = Helper.ToInt32(bn, 0); // byte[] bb = _strings.GetData(blknum, out _pagelistalllocblock); // keys = (object[])BJSON.ToObject(bb); //} for (int i = 0; i < count; i++) { int idx = index + _rowSize * i; byte ks = b[idx]; T key; if (_externalStrings == false) { key = _T.GetObject(b, idx + 1, ks); } else { if (keys == null) { key = _T.GetObject(b, idx + 1, ks); // do old way until better way } else { key = (T)keys[i]; } } int pagenum = Helper.ToInt32(b, idx + 1 + _maxKeySize); // add counts int unique = Helper.ToInt32(b, idx + 1 + _maxKeySize + 4); // FEATURE : add dup count PageList.Add(key, new PageInfo(pagenum, unique, 0)); } } else { throw new Exception("Page List header is invalid"); } return(nextpage); } }