private static MGRB DoBitOperation(MGRB bits, MGRB c, OPERATION op, int maxsize)
{
if (bits != null)
{
switch (op)
{
case OPERATION.AND:
bits = bits.And(c);
break;
case OPERATION.OR:
bits = bits.Or(c);
break;
case OPERATION.ANDNOT:
bits = bits.And(c.Not(maxsize));
break;
}
}
else
{
bits = c;
}
return(bits);
}
private long SaveBitmapToFile(MGRB bmp)
{
long off = _lastBitmapOffset;
var dat = bmp.Serialize();
var hdr = new byte[_hdrlen];
var b = fastBinaryJSON.BJSON.ToBJSON(dat, new fastBinaryJSON.BJSONParameters {
UseExtensions = false
});
hdr[0] = (byte)'b';
hdr[1] = (byte)'m';
hdr[2] = 0; // uncompressed
if (Global.CompressBitmapBytes)
{
hdr[2] = 1;
b = MiniLZO.Compress(b);
}
var s = Helper.GetBytes(b.Length, false);
Buffer.BlockCopy(s, 0, hdr, 3, 4);
_bitmapFileWrite.Write(hdr, 0, hdr.Length);
_lastBitmapOffset += hdr.Length;
_bitmapFileWrite.Write(b, 0, b.Length);
_lastBitmapOffset += b.Length;
return(off);
}
private MGRB LoadBitmap(long offset)
{
MGRB bc = new MGRB();
if (offset == -1)
{
return(bc);
}
FileStream bmp = _bitmapFileRead;
bmp.Seek(offset, SeekOrigin.Begin);
var hdr = new byte[_hdrlen];
bmp.Read(hdr, 0, hdr.Length);
if (hdr[0] == (byte)'b' && hdr[1] == (byte)'m')
{
int c = Helper.ToInt32(hdr, 3);
var b = new byte[c];
bmp.Read(b, 0, c);
if (hdr[2] == 1)
{
b = MiniLZO.Decompress(b);
}
bc.Deserialize(fastBinaryJSON.BJSON.ToObject <MGRBData>(b));
}
else
{
log.Error("bitmap not recognized");
}
return(bc);
}
private MGRB internalGetBitmap(int recno)
{
lock (_readlock)
{
MGRB ba = new MGRB();
if (recno == -1)
{
return(ba);
}
if (_cache.TryGetValue(recno, out ba))
{
return(ba);
}
else
{
long offset = 0;
//if (_offsetCache.TryGetValue(recno, out offset) == false)
{
offset = ReadRecordOffset(recno);
// _offsetCache.Add(recno, offset);
}
ba = LoadBitmap(offset);
_cache.Add(recno, ba);
return(ba);
}
}
}
public MGRB And(MGRB B)
{
var v = new SafeSortedList <int, Container>();
var len = _size;
if (B.Length < len)
{
len = B.Length;
}
var a = LastContainerIdx();
var b = B.LastContainerIdx();
var min = a;
if (b < min)
{
min = b;
}
min++;
for (int i = 0; i < min; i++)
{
Container ca = null;
Container cb = null;
_containers.TryGetValue(i, out ca);
B._containers.TryGetValue(i, out cb);
if (ca != null && cb != null)
{
v.Add(i, containerAND(ca, cb));
}
}
return(new MGRB(v, len));
}
public IEnumerable <int> FindRows(string filter)
{
checkloaded();
MGRB bits = ExecutionPlan(filter, _docs.RecordCount());
// enumerate records
return(bits.GetBitIndexes());
}
private void ReadFile()
{
byte[] b = File.ReadAllBytes(_path + _filename);
var o = fastBinaryJSON.BJSON.ToObject <MGRBData>(b);
_bits = new MGRB();
_bits.Deserialize(o);
}
private void InitializeFreeList()
{
if (_lastBlockNumber < 0)
{
// write master block
_datawrite.Write(new byte[_BLOCKSIZE], 0, _BLOCKSIZE);
_lastBlockNumber = 1;
//_masterblock.Add("freelist", -1);
}
else
{
_freeList = new MGRB();
// read master block data
var b = ReadBlock(0);
if (b[0] == (byte)'F' && b[1] == (byte)'L')
{
// get free block num and size
int block = Helper.ToInt32(b, 2);
int len = Helper.ToInt32(b, 2 + 4);
int freeblock = block;
b = new byte[len];
var offset = 0;
bool failed = false;
// read blocks upto size from block num
SeekBlock(block);
while (len > 0)
{
// check header
var bb = ReadBlock(block++);
if (bb[0] != (byte)'F' || bb[1] != (byte)'L')
{
// throw exception??
_log.Error("Free list header does not match : " + _filename);
failed = true;
break;
}
int c = len > _BLOCKSIZE ? _BLOCKSIZE - 2 : len;
Buffer.BlockCopy(bb, 2, b, offset, c);
len -= c;
offset += c;
}
if (failed == false)
{
// read freelist from master block from end of file
var o = fastBinaryJSON.BJSON.ToObject <MGRBData>(b);
_freeList.Deserialize(o);
// truncate end of file freelist blocks if lastblock < file size
if (_datawrite.Length > _lastBlockNumber * _BLOCKSIZE)
{
_datawrite.SetLength(_lastBlockNumber * _BLOCKSIZE);
}
}
_lastBlockNumber = freeblock;
}
}
}
public MGRB AndNot(MGRB b)
{
long c = _size;
if (b._size > c)
{
c = b._size;
}
return(And(b.Not(c)));
}
public MGRB Copy()
{
if (_containers.Count() > 0)
{
var o = Serialize();
var m = new MGRB();
m.Deserialize(o);
return(m);
}
return(new MGRB());
}
private void doPageOperation(ref MGRB res, int pageidx)
{
Page <T> page = LoadPage(_pageList.GetValue(pageidx).PageNumber);
T[] keys = page.tree.Keys(); // avoid sync issues
foreach (var k in keys)
{
int bn = page.tree[k].DuplicateBitmapNumber;
res = res.Or(_index.GetDuplicateBitmap(bn));
}
}
public void SetDuplicate(int bitmaprecno, int record)
{
using (new L(this))
{
MGRB ba = null;
ba = internalGetBitmap(bitmaprecno); //GetBitmap(bitmaprecno);
ba.Set(record, true);
_isDirty = true;
}
}
private MGRB doLessOp(RDBExpression exp, T key)
{
bool found = false;
int pos = FindPageOrLowerPosition(key, ref found);
MGRB result = new MGRB();
if (pos > 0)
{
// all the pages before
for (int i = 0; i < pos - 1; i++)
{
doPageOperation(ref result, i);
}
}
// key page
Page <T> page = LoadPage(_pageList.GetValue(pos).PageNumber);
T[] keys = page.tree.Keys();
Array.Sort(keys);
// find better end position rather than last key
pos = Array.BinarySearch <T>(keys, key);
if (pos < 0)
{
pos = ~pos;
}
for (int i = 0; i < pos; i++)
{
T k = keys[i];
if (k.CompareTo(key) > 0)
{
break;
}
int bn = page.tree[k].DuplicateBitmapNumber;
if (k.CompareTo(key) < 0)
{
result = result.Or(_index.GetDuplicateBitmap(bn));
}
if (exp == RDBExpression.LessEqual && k.CompareTo(key) == 0)
{
result = result.Or(_index.GetDuplicateBitmap(bn));
}
}
return(result);
}
private void SaveBitmap(int recno, MGRB bmp)
{
lock (_writelock)
{
long offset = SaveBitmapToFile(bmp);
//long v;
//if (_offsetCache.TryGetValue(recno, out v))
// _offsetCache[recno] = offset;
//else
// _offsetCache.Add(recno, offset);
long pointer = ((long)recno) * 8;
_recordFileWrite.Seek(pointer, SeekOrigin.Begin);
byte[] b = new byte[8];
b = Helper.GetBytes(offset, false);
_recordFileWrite.Write(b, 0, 8);
}
}
public IEnumerable <string> FindDocumentFileNames(string filter)
{
checkloaded();
MGRB bits = ExecutionPlan(filter, _docs.RecordCount());
// enumerate documents
foreach (int i in bits.GetBitIndexes())
{
if (i > _lastDocNum - 1)
{
break;
}
string b = _docs.ReadData(i);
var d = (Dictionary <string, object>)fastJSON.JSON.Parse(b);
yield return(d["FileName"].ToString());
}
}
public MGRB Or(MGRB B)
{
var v = new SafeSortedList <int, Container>();
var len = _size;
if (B.Length > len)
{
len = B.Length;
}
var a = LastContainerIdx();
var b = B.LastContainerIdx();
var max = a;
if (b > max)
{
max = b;
}
max++;
for (int i = 0; i < max; i++)
{
Container ca = null;
Container cb = null;
_containers.TryGetValue(i, out ca);
B._containers.TryGetValue(i, out cb);
if (ca == null && cb != null)
{
v.Add(i, cb.Copy());
}
else if (cb == null && ca != null)
{
v.Add(i, ca.Copy());
}
else if (ca != null && cb != null)
{
v.Add(i, containerOR(ca, cb));
}
}
return(new MGRB(v, len));
}
public IEnumerable <T> FindDocuments <T>(string filter)
{
checkloaded();
MGRB bits = ExecutionPlan(filter, _docs.RecordCount());
// enumerate documents
foreach (int i in bits.GetBitIndexes())
{
if (i > _lastDocNum - 1)
{
break;
}
string b = _docs.ReadData(i);
T d = fastJSON.JSON.ToObject <T>(b, new fastJSON.JSONParameters {
ParametricConstructorOverride = true
});
yield return(d);
}
}
public void Commit(bool freeMemory)
{
if (_isDirty == false)
{
return;
}
using (new L(this))
{
log.Debug("writing " + _filename);
int[] keys = _cache.Keys();
Array.Sort(keys);
foreach (int k in keys)
{
MGRB bmp = null;
if (_cache.TryGetValue(k, out bmp) && bmp.isDirty)
{
bmp.Optimize();
SaveBitmap(k, bmp);
bmp.isDirty = false;
}
}
Flush();
if (freeMemory)
{
if (Global.UseLessMemoryStructures)
{
_cache = new SafeSortedList <int, MGRB>();
}
else
{
_cache = new SafeDictionary <int, MGRB>();
}
log.Debug(" freeing cache");
}
_isDirty = false;
}
}
public void InPlaceOR(MGRB left)
{
lock (_lock)
_bits = _bits.Or(left);
}
public MGRB Query(RDBExpression ex, object from, int maxsize)
{
// always return everything
return(MGRB.Fill(maxsize));
}
private MGRB ExecutionPlan(string filter, int maxsize)
{
//_log.Debug("query : " + filter);
DateTime dt = FastDateTime.Now;
// query indexes
string[] words = filter.Split(' ');
//bool defaulttoand = true;
//if (filter.IndexOfAny(new char[] { '+', '-' }, 0) > 0)
// defaulttoand = false;
MGRB found = null;// MGRB.Fill(maxsize);
foreach (string s in words)
{
int c;
bool not = false;
string word = s;
if (s == "")
{
continue;
}
OPERATION op = OPERATION.AND;
//if (defaulttoand)
// op = OPERATION.AND;
if (word.StartsWith("+"))
{
op = OPERATION.OR;
word = s.Replace("+", "");
}
if (word.StartsWith("-"))
{
op = OPERATION.ANDNOT;
word = s.Replace("-", "");
not = true;
if (found == null) // leading with - -> "-oak hill"
{
found = MGRB.Fill(maxsize);
}
}
if (word.Contains("*") || word.Contains("?"))
{
MGRB wildbits = new MGRB();
// do wildcard search
Regex reg = new Regex("^" + word.Replace("*", ".*").Replace("?", ".") + "$", RegexOptions.IgnoreCase);
foreach (string key in _words.Keys())
{
if (reg.IsMatch(key))
{
_words.TryGetValue(key, out c);
MGRB ba = _bitmaps.GetBitmap(c);
wildbits = DoBitOperation(wildbits, ba, OPERATION.OR, maxsize);
}
}
if (found == null)
{
found = wildbits;
}
else
{
if (not) // "-oak -*l"
{
found = found.AndNot(wildbits);
}
else if (op == OPERATION.AND)
{
found = found.And(wildbits);
}
else
{
found = found.Or(wildbits);
}
}
}
else if (_words.TryGetValue(word.ToLowerInvariant(), out c))
{
// bits logic
MGRB ba = _bitmaps.GetBitmap(c);
found = DoBitOperation(found, ba, op, maxsize);
}
else if (op == OPERATION.AND)
{
found = new MGRB();
}
}
if (found == null)
{
return(new MGRB());
}
// remove deleted docs
MGRB ret;
if (_docMode)
{
ret = found.AndNot(_deleted.GetBits());
}
else
{
ret = found;
}
//_log.Debug("query time (ms) = " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
return(ret);
}
public MGRB Query(object fromkey, object tokey, int maxsize)
{
return(MGRB.Fill(maxsize)); // TODO : all or none??
}
private void RebuildDataFiles()
{
MGIndex <string> keys = null;
try
{
// remove old free list
if (File.Exists(_Path + "data.bmp"))
{
File.Delete(_Path + "data.bmp");
}
_datastore = new StorageFileHF(_Path + "data.mghf", Global.HighFrequencyKVDiskBlockSize);
_BlockSize = _datastore.GetBlockSize();
if (File.Exists(_Path + "keys.idx"))
{
_log.Debug("removing old keys index");
foreach (var f in Directory.GetFiles(_Path, "keys.*"))
{
File.Delete(f);
}
}
keys = new MGIndex <string>(_Path, "keys.idx", 255, /*Global.PageItemCount,*/ false);
MGRB visited = new MGRB();
int c = _datastore.NumberofBlocks();
for (int i = 1; i < c; i++) // go through blocks skip first
{
if (visited.Get(i))
{
continue;
}
byte[] b = _datastore.ReadBlockBytes(i, _blockheader.Length + 255);
int bnum = Helper.ToInt32(b, 0);
if (bnum > 0) // check if a start block
{
visited.Set(i, true);
_datastore.FreeBlock(i); // mark as free
continue;
}
AllocationBlock ab = new AllocationBlock();
// start block found
int blocknumexpected = 0;
int next = ParseBlockHeader(ab, b, blocknumexpected);
int last = 0;
bool freelast = false;
AllocationBlock old = null;
if (ab.key == null)
{
continue;
}
if (keys.Get(ab.key, out last))
{
old = this.FillAllocationBlock(last);
freelast = true;
}
blocknumexpected++;
bool failed = false;
if (ab.deleteKey == false)
{
while (next > 0) // read the blocks
{
ab.Blocks.Add(next);
b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen);
next = ParseBlockHeader(ab, b, blocknumexpected);
if (next == -1) // non matching block
{
failed = true;
break;
}
blocknumexpected++;
}
}
else
{
failed = true;
keys.RemoveKey(ab.key);
}
// new data ok
if (failed == false)
{
keys.Set(ab.key, i); // valid block found
if (freelast && old != null) // free the old blocks
{
_datastore.FreeBlocks(old.Blocks);
}
}
visited.Set(i, true);
}
// all ok delete temp.$ file
if (File.Exists(_Path + _dirtyFilename))
{
File.Delete(_Path + _dirtyFilename);
}
}
catch (Exception ex)
{
_log.Error(ex);
}
finally
{
_log.Debug("Shutting down files and index");
_datastore.Shutdown();
keys.SaveIndex();
keys.Shutdown();
}
}
public MGRB Query(T from, T to, int maxsize)
{
MGRB bits = new MGRB();
T temp = default(T);
if (from.CompareTo(to) > 0) // check values order
{
temp = from;
from = to;
to = temp;
}
// find first page and do > than
bool found = false;
int startpos = FindPageOrLowerPosition(from, ref found);
// find last page and do < than
int endpos = FindPageOrLowerPosition(to, ref found);
bool samepage = startpos == endpos;
// from key page
Page <T> page = LoadPage(_pageList.GetValue(startpos).PageNumber);
T[] keys = page.tree.Keys();
Array.Sort(keys);
// find better start position rather than 0
int pos = Array.BinarySearch <T>(keys, from); // FEATURE : rewrite??
if (pos < 0)
{
pos = ~pos;
}
for (int i = pos; i < keys.Length; i++)
{
T k = keys[i];
int bn = page.tree[k].DuplicateBitmapNumber;
if (samepage)
{
if (k.CompareTo(from) >= 0 && k.CompareTo(to) <= 0) // if from,to same page
{
bits = bits.Or(_index.GetDuplicateBitmap(bn));
}
}
else
{
if (k.CompareTo(from) >= 0)
{
bits = bits.Or(_index.GetDuplicateBitmap(bn));
}
}
}
if (!samepage)
{
// to key page
page = LoadPage(_pageList.GetValue(endpos).PageNumber);
keys = page.tree.Keys();
Array.Sort(keys);
// find better end position rather than last key
pos = Array.BinarySearch <T>(keys, to);
if (pos < 0)
{
pos = ~pos;
}
for (int i = 0; i <= pos; i++)
{
T k = keys[i];
int bn = page.tree[k].DuplicateBitmapNumber;
if (k.CompareTo(to) <= 0)
{
bits = bits.Or(_index.GetDuplicateBitmap(bn));
}
}
// do all pages in between
for (int i = startpos + 1; i < endpos; i++)
{
doPageOperation(ref bits, i);
}
}
return(bits);
}