private void CreatePageListData(SafeSortedList <T, PageInfo> _pages, int offset, int rowindex, int counter, byte[] page, int blocknum)
{
int idx = rowindex + _rowSize * counter;
// key bytes
byte[] kk;
byte size;
if (_externalStrings == false)
{
kk = _T.GetBytes(_pages.GetKey(counter + offset));
size = (byte)kk.Length;
if (size > _maxKeySize)
{
size = _maxKeySize;
}
}
else
{
kk = new byte[4];
Buffer.BlockCopy(Helper.GetBytes(counter + offset, false), 0, kk, 0, 4);
size = 4;
}
// key size = 1 byte
page[idx] = size;
Buffer.BlockCopy(kk, 0, page, idx + 1, page[idx]);
// offset = 4 bytes
byte[] b = Helper.GetBytes(_pages.GetValue(offset + counter).PageNumber, false);
Buffer.BlockCopy(b, 0, page, idx + 1 + _maxKeySize, b.Length);
// add counts
b = Helper.GetBytes(_pages.GetValue(offset + counter).UniqueCount, false);
Buffer.BlockCopy(b, 0, page, idx + 1 + _maxKeySize + 4, b.Length);
// FEATURE : add dup counts
}
public void Commit(bool freeMemory)
{
if (_isDirty == false)
{
return;
}
using (new L(this))
{
log.Debug("writing " + _FileName);
int[] keys = _cache.Keys();
Array.Sort(keys);
foreach (int k in keys)
{
WAHBitArray bmp = null;
if (_cache.TryGetValue(k, out bmp) && bmp.isDirty)
{
SaveBitmap(k, bmp);
bmp.FreeMemory();
bmp.isDirty = false;
}
}
Flush();
if (freeMemory)
{
_cache = //new SafeDictionary<int, WAHBitArray>();
new SafeSortedList <int, WAHBitArray>();
log.Debug(" freeing cache");
}
_isDirty = false;
}
}
public WAHBitArray(TYPE type, uint[] ints)
{
_state = type;
switch (type)
{
case TYPE.WAH:
_compressed = ints;
Uncompress();
_state = TYPE.Bitarray;
_compressed = null;
break;
case TYPE.Bitarray:
_uncompressed = ints;
break;
case TYPE.Indexes:
if (Global.UseLessMemoryStructures)
{
_offsets = new SafeSortedList <uint, bool>();
}
else
{
_offsets = new SafeDictionary <uint, bool>();
}
//new Dictionary<uint, bool>();
foreach (var i in ints)
{
_offsets.Add(i, true);
}
break;
}
}
public static MGRB Fill(long count)
{
if (count == 0)
{
return(new MGRB());
}
var con = new SafeSortedList <int, Container>();
int i = 0;
long c = count;
while (count > 0)
{
if (count > Container.BSize)
{
con.Add(i, new BitmapContainer(true));
}
else
{
con.Add(i, new BitmapContainer((int)count));
}
count -= Container.BSize;
i++;
}
return(new MGRB(con, c));
}
private void LoadWords()
{
lock (_lock)
{
if (_words == null)
{
_words = new SafeSortedList <string, int>();
}
if (File.Exists(_Path + _FileName + ".words") == false)
{
return;
}
// load words
byte[] b = File.ReadAllBytes(_Path + _FileName + ".words");
if (b.Length == 0)
{
return;
}
MemoryStream ms = new MemoryStream(b);
BinaryReader br = new BinaryReader(ms, Encoding.UTF8);
string s = br.ReadString();
while (s != "")
{
int off = br.ReadInt32();
_words.Add(s, off);
try
{
s = br.ReadString();
}
catch { s = ""; }
}
_log.Debug("Word Count = " + _words.Count);
_loaded = true;
}
}
public MGRB And(MGRB B)
{
var v = new SafeSortedList <int, Container>();
var len = _size;
if (B.Length < len)
{
len = B.Length;
}
var a = LastContainerIdx();
var b = B.LastContainerIdx();
var min = a;
if (b < min)
{
min = b;
}
min++;
for (int i = 0; i < min; i++)
{
Container ca = null;
Container cb = null;
_containers.TryGetValue(i, out ca);
B._containers.TryGetValue(i, out cb);
if (ca != null && cb != null)
{
v.Add(i, containerAND(ca, cb));
}
}
return(new MGRB(v, len));
}
public MGRB Not()
{
var con = new SafeSortedList <int, Container>();
foreach (var c in _containers)
{
con.Add(c.Key, c.Value.Not());
}
return(new MGRB(con, _size));
}
internal MGRB(SafeSortedList <int, Container> containers)
{
_containers = containers;
var k = _containers.Keys();
var l = k.Length - 1;
if (l >= 0)
{
_size = (k[l] << 16) + _containers.GetValue(l).Size;
}
}
internal void SavePageList(SafeSortedList <T, PageInfo> _pages, List <int> diskpages)
{
lock (_fileLock)
{
T[] keys = _pages.Keys();
int blocknum = 0;
// TODO : needed??
//if (_externalStrings)
//{
// if (_pagelistalllocblock != null)
// _strings.FreeBlocks(_pagelistalllocblock);
// blocknum = _strings.SaveData("pagelist", BJSON.ToBJSON(keys,
// new BJSONParameters { UseUnicodeStrings = false, UseTypedArrays = false }));
// File.WriteAllBytes(_FileName + ".pagelist", Helper.GetBytes(blocknum, false));
//}
// save page list
int c = (_pages.Count() / Global.PageItemCount) + 1;
// allocate pages needed
while (c > diskpages.Count)
{
diskpages.Add(GetNewPageNumber());
}
byte[] page = new byte[_PageLength];
for (int i = 0; i < (diskpages.Count - 1); i++)
{
byte[] block = CreateBlockHeader(1, Global.PageItemCount, diskpages[i + 1]);
Buffer.BlockCopy(block, 0, page, 0, block.Length);
for (int j = 0; j < Global.PageItemCount; j++)
{
CreatePageListData(_pages, i * Global.PageItemCount, block.Length, j, page, blocknum);
}
SeekPage(diskpages[i]);
_file.Write(page, 0, page.Length);
}
c = _pages.Count() % Global.PageItemCount;
byte[] lastblock = CreateBlockHeader(1, (ushort)c, -1);
Buffer.BlockCopy(lastblock, 0, page, 0, lastblock.Length);
int lastoffset = (_pages.Count() / Global.PageItemCount) * Global.PageItemCount;
for (int j = 0; j < c; j++)
{
CreatePageListData(_pages, lastoffset, lastblock.Length, j, page, blocknum);
}
SeekPage(diskpages[diskpages.Count - 1]);
_file.Write(page, 0, page.Length);
}
}
public WAHBitArray()
{
_state = TYPE.Indexes;
if (Global.UseLessMemoryStructures)
{
_offsets = new SafeSortedList <uint, bool>();
}
else
{
_offsets = new SafeDictionary <uint, bool>();
}
}
internal void SavePageList(SafeSortedList <T, PageInfo> _pages, List <int> diskpages)
{
lock (_fileLock)
{
// save page list
int c = (_pages.Count / Global.PageItemCount) + 1;
// allocate pages needed
while (c > diskpages.Count)
{
diskpages.Add(GetNewPageNumber());
}
byte[] page = new byte[_PageLength];
for (int i = 0; i < (diskpages.Count - 1); i++)
{
byte[] block = CreateBlockHeader(1, Global.PageItemCount, diskpages[i + 1]);
Buffer.BlockCopy(block, 0, page, 0, block.Length);
for (int j = 0; j < Global.PageItemCount; j++)
{
CreatePageListData(_pages, i * Global.PageItemCount, block.Length, j, page);
}
SeekPage(diskpages[i]);
_file.Write(page, 0, page.Length);
}
c = _pages.Count % Global.PageItemCount;
byte[] lastblock = CreateBlockHeader(1, (ushort)c, -1);
Buffer.BlockCopy(lastblock, 0, page, 0, lastblock.Length);
int lastoffset = (_pages.Count / Global.PageItemCount) * Global.PageItemCount;
for (int j = 0; j < c; j++)
{
CreatePageListData(_pages, lastoffset, lastblock.Length, j, page);
}
// [BRADS] This is null when shutting down sometimes
if (_file == null)
{
return;
}
SeekPage(diskpages[diskpages.Count - 1]);
_file.Write(page, 0, page.Length);
}
}
internal MGRB(SafeSortedList <int, Container> containers, long size)
{
_containers = containers;
var k = _containers.Keys();
_size = size;
if (size <= 0)//== -1)
{
_size = 0;
var l = k.Length - 1;
if (l >= 0)
{
_size = (k[l] << 16) + _containers.GetValue(l).Size;
}
}
}
public void GetPageList(List <int> PageListDiskPages, SafeSortedList <T, PageInfo> PageList, out int lastIndexedRow)
{
lastIndexedRow = Helper.ToInt32(_FileHeader, 11);
// load page list
PageListDiskPages.Add(0); // first page list
int nextpage = LoadPageListData(0, PageList);
while (nextpage != -1)
{
nextpage = LoadPageListData(nextpage, PageList);
if (nextpage != -1)
{
PageListDiskPages.Add(nextpage);
}
}
}
public static MGRB Fill(long count)
{
if (count == 0)
{
return(new MGRB());
}
var con = new SafeSortedList <int, Container>();
var c = count >> 16;
for (int i = 0; i <= c; i++)
{
con.Add(i, new BitmapContainer(true));
}
return(new MGRB(con));
}
public MGRB Or(MGRB B)
{
var v = new SafeSortedList <int, Container>();
var len = _size;
if (B.Length > len)
{
len = B.Length;
}
var a = LastContainerIdx();
var b = B.LastContainerIdx();
var max = a;
if (b > max)
{
max = b;
}
max++;
for (int i = 0; i < max; i++)
{
Container ca = null;
Container cb = null;
_containers.TryGetValue(i, out ca);
B._containers.TryGetValue(i, out cb);
if (ca == null && cb != null)
{
v.Add(i, cb.Copy());
}
else if (cb == null && ca != null)
{
v.Add(i, ca.Copy());
}
else if (ca != null && cb != null)
{
v.Add(i, containerOR(ca, cb));
}
}
return(new MGRB(v, len));
}
public BitmapIndex(string path, string filename)
{
if (Global.UseLessMemoryStructures)
{
_cache = new SafeSortedList <int, WAHBitArray>();
}
else
{
_cache = new SafeDictionary <int, WAHBitArray>();
}
_FileName = Path.GetFileNameWithoutExtension(filename);
_Path = path;
if (_Path.EndsWith(Path.DirectorySeparatorChar.ToString()) == false)
{
_Path += Path.DirectorySeparatorChar.ToString();
}
Initialize();
}
public MGRB Not(long count)
{
var con = new SafeSortedList <int, Container>();
var c = count >> 16;
for (int i = 0; i <= c; i++)
{
Container a = null;
_containers.TryGetValue(i, out a);
if (a == null)
{
con.Add(i, new BitmapContainer(true));
}
else
{
con.Add(i, a.Not());
}
}
return(new MGRB(con, count));
}
private int LoadPageListData(int page, SafeSortedList <T, PageInfo> PageList)
{
lock (_fileLock)
{
// load page list data
int nextpage = -1;
SeekPage(page);
byte[] b = new byte[_PageLength];
_file.Read(b, 0, _PageLength);
if (b[0] == _BlockHeader[0] && b[1] == _BlockHeader[1] && b[2] == _BlockHeader[2] && b[3] == _BlockHeader[3])
{
short count = Helper.ToInt16(b, 5);
if (count > _PageNodeCount)
{
throw new Exception("Count > node size");
}
nextpage = Helper.ToInt32(b, 11);
int index = _BlockHeader.Length;
for (int i = 0; i < count; i++)
{
int idx = index + _rowSize * i;
byte ks = b[idx];
T key = _T.GetObject(b, idx + 1, ks);
int pagenum = Helper.ToInt32(b, idx + 1 + _maxKeySize);
// add counts
int unique = Helper.ToInt32(b, idx + 1 + _maxKeySize + 4);
// FEATURE : add dup count
PageList.Add(key, new PageInfo(pagenum, unique, 0));
}
}
else
{
throw new Exception("Page List header is invalid");
}
return(nextpage);
}
}
private void CreatePageListData(SafeSortedList <T, PageInfo> _pages, int i, byte[] page, int index, int j)
{
int idx = index + _rowSize * j;
// key bytes
byte[] kk = _T.GetBytes(_pages.GetKey(j + i));
byte size = (byte)kk.Length;
if (size > _maxKeySize)
{
size = _maxKeySize;
}
// key size = 1 byte
page[idx] = size;
Buffer.BlockCopy(kk, 0, page, idx + 1, page[idx]);
// offset = 4 bytes
byte[] b = Helper.GetBytes(_pages.GetValue(i + j).PageNumber, false);
Buffer.BlockCopy(b, 0, page, idx + 1 + _maxKeySize, b.Length);
// add counts
b = Helper.GetBytes(_pages.GetValue(i + j).UniqueCount, false);
Buffer.BlockCopy(b, 0, page, idx + 1 + _maxKeySize + 4, b.Length);
// FEATURE : add dup counts
}
internal MGRB(SafeSortedList <int, Container> containers) : this(containers, -1)
{
}
private int LoadPageListData(int page, SafeSortedList <T, PageInfo> PageList)
{
lock (_fileLock)
{
// load page list data
int nextpage = -1;
SeekPage(page);
byte[] b = new byte[_PageLength];
_file.Read(b, 0, _PageLength);
if (b[0] == _BlockHeader[0] && b[1] == _BlockHeader[1] && b[2] == _BlockHeader[2] && b[3] == _BlockHeader[3])
{
short count = Helper.ToInt16(b, 5);
if (count > _PageNodeCount)
{
throw new Exception("Count > node size");
}
nextpage = Helper.ToInt32(b, 11);
int index = _BlockHeader.Length;
object[] keys = null;
// TODO : needed??
//if (File.Exists(_FileName + ".pagelist"))
//{
// var bn = File.ReadAllBytes(_FileName + ".pagelist");
// int blknum = Helper.ToInt32(bn, 0);
// byte[] bb = _strings.GetData(blknum, out _pagelistalllocblock);
// keys = (object[])BJSON.ToObject(bb);
//}
for (int i = 0; i < count; i++)
{
int idx = index + _rowSize * i;
byte ks = b[idx];
T key;
if (_externalStrings == false)
{
key = _T.GetObject(b, idx + 1, ks);
}
else
{
if (keys == null)
{
key = _T.GetObject(b, idx + 1, ks); // do old way until better way
}
else
{
key = (T)keys[i];
}
}
int pagenum = Helper.ToInt32(b, idx + 1 + _maxKeySize);
// add counts
int unique = Helper.ToInt32(b, idx + 1 + _maxKeySize + 4);
// FEATURE : add dup count
PageList.Add(key, new PageInfo(pagenum, unique, 0));
}
}
else
{
throw new Exception("Page List header is invalid");
}
return(nextpage);
}
}
