/// <summary>
/// 内部获取位图
/// </summary>
/// <param name="recno"></param>
/// <returns></returns>
private WAHBitArray internalGetBitmap(int recno)
{
lock (_readlock)
{
WAHBitArray ba = new WAHBitArray();
if (recno == -1)//-1等于不存在，立即创建返回
{
return(ba);
}
if (_cache.TryGetValue(recno, out ba))
{
return(ba);
}
else
{
long offset = 0;
//if (_offsetCache.TryGetValue(recno, out offset) == false)
{
offset = ReadRecordOffset(recno);
// _offsetCache.Add(recno, offset);
}
ba = LoadBitmap(offset);
_cache.Add(recno, ba);
return(ba);
}
}
}
private static WAHBitArray DoBitOperation(WAHBitArray bits, WAHBitArray c, OPERATION op, int maxsize)
{
if (bits != null)
{
switch (op)
{
case OPERATION.AND:
bits = bits.And(c);
break;
case OPERATION.OR:
bits = bits.Or(c);
break;
case OPERATION.ANDNOT:
bits = bits.And(c.Not(maxsize));
break;
}
}
else
{
bits = c;
}
return(bits);
}
/// <summary>
///
/// </summary>
/// <param name="freeMemory">是否释放内存</param>
public void Commit(bool freeMemory)
{
if (_isDirty == false)
{
return;
}
using (new L(this))
{
log.Debug("writing " + _FileName);
int[] keys = _cache.Keys();
Array.Sort(keys);
foreach (int k in keys)
{
WAHBitArray bmp = null;
if (_cache.TryGetValue(k, out bmp) && bmp.isDirty)
{
this.SaveBitmap(k, bmp);
bmp.FreeMemory();
bmp.isDirty = false;
}
}
Flush();
if (freeMemory)
{
_cache = //new SafeDictionary<int, WAHBitArray>();
new SafeSortedList <int, WAHBitArray>();
log.Debug(" freeing cache");
}
_isDirty = false;
}
}
/// <summary>
/// 保存到文件
/// -----------------------------------------------------------------
/// BITMAP FILE FORMAT
/// 0 'B','M'
/// 2 uint count = 4 bytes
/// 6 Bitmap type :
/// 0 = int record list
/// 1 = uint bitmap
/// 2 = rec# indexes
/// 7 '0'
/// 8 uint data
/// -----------------------------------------------------------------
/// </summary>
/// <param name="bmp"></param>
/// <returns>文件中的位置</returns>
private long SaveBitmapToFile(WAHBitArray bmp)
{
long off = _lastBitmapOffset;
WAHBitArray.TYPE t;
uint[] bits = bmp.GetCompressed(out t);
byte[] b = new byte[bits.Length * 4 + 8];
// write header data
b[0] = ((byte)'B');
b[1] = ((byte)'M');
Buffer.BlockCopy(Helper.GetBytes(bits.Length, false), 0, b, 2, 4);
b[6] = (byte)t;
b[7] = (byte)(0);
for (int i = 0; i < bits.Length; i++)
{
byte[] u = Helper.GetBytes((int)bits[i], false);
Buffer.BlockCopy(u, 0, b, i * 4 + 8, 4);
}
_bitmapFileWrite.Write(b, 0, b.Length);
_lastBitmapOffset += b.Length;
return(off);
}
public IEnumerable <int> FindRows(string filter)
{
checkloaded();
WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount());
// enumerate records
return(bits.GetBitIndexes());
}
/// <summary>
///
/// </summary>
/// <param name="bitmaprecno">索引编号</param>
/// <param name="record">文档编号</param>
public void SetDuplicate(int bitmaprecno, int record)
{
using (new L(this))
{
WAHBitArray ba = null;
ba = internalGetBitmap(bitmaprecno); //GetBitmap(bitmaprecno);
ba.Set(record, true);
_isDirty = true;
}
}
private void doPageOperation(ref WAHBitArray res, int pageidx)
{
Page <T> page = LoadPage(_pageList.GetValue(pageidx).PageNumber);
T[] keys = page.tree.Keys(); // avoid sync issues
foreach (var k in keys)
{
int bn = page.tree[k].DuplicateBitmapNumber;
res = res.Or(_index.GetDuplicateBitmap(bn));
}
}
public WAHBitArray Xor(WAHBitArray op)
{
lock (_lock)
{
uint[] left;
uint[] right;
prelogic(op, out left, out right);
for (int i = 0; i < left.Length; i++)
{
left[i] ^= right[i];
}
return(new WAHBitArray(TYPE.Bitarray, left));
}
}
internal void Initialize()
{
if (_readfreeList != null)
{
_freeList = _readfreeList();
}
else
{
_freeList = new WAHBitArray();
if (File.Exists(_Path + _filename + ".free"))
{
ReadFreeListBMPFile(_Path + _filename + ".free");
// delete file so if failure no big deal on restart
File.Delete(_Path + _filename + ".free");
}
}
}
private void SaveBitmap(int recno, WAHBitArray bmp)
{
lock (_writelock)
{
long offset = SaveBitmapToFile(bmp);
//long v;
//if (_offsetCache.TryGetValue(recno, out v))
// _offsetCache[recno] = offset;
//else
// _offsetCache.Add(recno, offset);
long pointer = ((long)recno) * 8;
_recordFileWrite.Seek(pointer, SeekOrigin.Begin);
byte[] b = new byte[8];
b = Helper.GetBytes(offset, false);
_recordFileWrite.Write(b, 0, 8);
}
}
private WAHBitArray doMoreOp(RDBExpression exp, T key)
{
bool found = false;
int pos = FindPageOrLowerPosition(key, ref found);
WAHBitArray result = new WAHBitArray();
if (pos < _pageList.Count)
{
// all the pages after
for (int i = pos + 1; i < _pageList.Count; i++)
{
doPageOperation(ref result, i);
}
}
// key page
Page <T> page = LoadPage(_pageList.GetValue(pos).PageNumber);
T[] keys = page.tree.Keys();
Array.Sort(keys);
// find better start position rather than 0
pos = Array.IndexOf <T>(keys, key);
if (pos == -1)
{
pos = 0;
}
for (int i = pos; i < keys.Length; i++)
{
T k = keys[i];
int bn = page.tree[k].DuplicateBitmapNumber;
if (k.CompareTo(key) > 0)
{
result = result.Or(_index.GetDuplicateBitmap(bn));
}
if (exp == RDBExpression.GreaterEqual && k.CompareTo(key) == 0)
{
result = result.Or(_index.GetDuplicateBitmap(bn));
}
}
return(result);
}
private void ReadFreeListBMPFile(string filename)
{
byte[] b = File.ReadAllBytes(filename);
WAHBitArray.TYPE t = WAHBitArray.TYPE.WAH;
int j = 0;
if (b.Length % 4 > 0) // new format with the data type byte
{
t = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), b[0]);
j = 1;
}
List <uint> ints = new List <uint>();
for (int i = 0; i < b.Length / 4; i++)
{
ints.Add((uint)Helper.ToInt32(b, (i * 4) + j));
}
_freeList = new WAHBitArray(t, ints.ToArray());
}
public IEnumerable <string> FindDocumentFileNames(string filter)
{
checkloaded();
WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount());
// enumerate documents
foreach (int i in bits.GetBitIndexes())
{
if (i > _lastDocNum - 1)
{
break;
}
string b = _docs.ReadData(i);
//var d = (Dictionary<string, object>)fastJSON.JSON.Parse(b);//fastJSON
//yield return d["FileName"].ToString();
var d = _json.DeserializeObject <Document>(b);
yield return(d.FileName);
}
}
public IEnumerable <T> FindDocuments <T>(string filter)
{
checkloaded();
WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount());
// enumerate documents
foreach (int i in bits.GetBitIndexes())
{
if (i > _lastDocNum - 1)
{
break;
}
string b = _docs.ReadData(i);
//T d = fastJSON.JSON.ToObject<T>(b, new fastJSON.JSONParameters { ParametricConstructorOverride = true }); //fastJSON
T d = _json.DeserializeObject <T>(b);
yield return(d);
}
}
private void ReadFile()
{
byte[] b = File.ReadAllBytes(_path + _filename);
MemoryStream ms = new MemoryStream(b);
BinaryReader br = new BinaryReader(ms);
WAHBitArray.TYPE t = WAHBitArray.TYPE.WAH;
if (b.Length % 4 > 0) // new format with the data type byte
{
byte tb = br.ReadByte();
t = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), tb);
}
List <uint> ints = new List <uint>();
for (int i = 0; i < b.Length / 4; i++)
{
ints.Add((uint)br.ReadInt32());
}
_bits = new WAHBitArray(t, ints.ToArray());
}
private WAHBitArray doLessOp(RDBExpression exp, T key)
{
bool found = false;
int pos = FindPageOrLowerPosition(key, ref found);
WAHBitArray result = new WAHBitArray();
if (pos > 0)
{
// all the pages before
for (int i = 0; i < pos - 1; i++)
{
doPageOperation(ref result, i);
}
}
// key page
Page <T> page = LoadPage(_pageList.GetValue(pos).PageNumber);
T[] keys = page.tree.Keys();
Array.Sort(keys);
for (int i = 0; i < keys.Length; i++)
{
T k = keys[i];
if (k.CompareTo(key) > 0)
{
break;
}
int bn = page.tree[k].DuplicateBitmapNumber;
if (k.CompareTo(key) < 0)
{
result = result.Or(_index.GetDuplicateBitmap(bn));
}
if (exp == RDBExpression.LessEqual && k.CompareTo(key) == 0)
{
result = result.Or(_index.GetDuplicateBitmap(bn));
}
}
return(result);
}
private void prelogic(WAHBitArray op, out uint[] left, out uint[] right)
{
this.CheckBitArray();
left = this.GetBitArray();
right = op.GetBitArray();
int ic = left.Length;
int uc = right.Length;
if (ic > uc)
{
uint[] ar = new uint[ic];
right.CopyTo(ar, 0);
right = ar;
}
else if (ic < uc)
{
uint[] ar = new uint[uc];
left.CopyTo(ar, 0);
left = ar;
}
}
/// <summary>
/// 加载位图
/// </summary>
/// <param name="offset"></param>
/// <returns></returns>
private WAHBitArray LoadBitmap(long offset)
{
WAHBitArray bc = new WAHBitArray();
if (offset == -1)
{
return(bc);
}
List <uint> ar = new List <uint>();
WAHBitArray.TYPE type = WAHBitArray.TYPE.WAH;
FileStream bmp = _bitmapFileRead;
{
bmp.Seek(offset, SeekOrigin.Begin);
byte[] b = new byte[8];
bmp.Read(b, 0, 8); //读取头部
if (b[0] == (byte)'B' && b[1] == (byte)'M' && b[7] == 0) //验证
{
type = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), b[6]);
int c = Helper.ToInt32(b, 2);
byte[] buf = new byte[c * 4];
bmp.Read(buf, 0, c * 4);
for (int i = 0; i < c; i++)
{
ar.Add((uint)Helper.ToInt32(buf, i * 4));
}
}
}
bc = new WAHBitArray(type, ar.ToArray());
return(bc);
}
public WAHBitArray Query(RDBExpression ex, object from, int maxsize)
{
// always return everything
return(WAHBitArray.Fill(maxsize));
}
public void InPlaceOR(WAHBitArray left)
{
lock (_lock)
_bits = _bits.Or(left);
}
private void RebuildDataFiles()
{
MGIndex <string> keys = null;
try
{
// remove old free list
if (File.Exists(_Path + "data.bmp"))
{
File.Delete(_Path + "data.bmp");
}
_datastore = new StorageFileHF(Path.Combine(_Path, "data.mghf"), Global.HighFrequencyKVDiskBlockSize);
_BlockSize = _datastore.GetBlockSize();
if (File.Exists(_Path + "keys.idx"))
{
_log.Debug("removing old keys index");
foreach (var f in Directory.GetFiles(_Path, "keys.*"))
{
File.Delete(f);
}
}
keys = new MGIndex <string>(_Path, "keys.idx", 255, /*Global.PageItemCount,*/ false);
WAHBitArray visited = new WAHBitArray();
int c = _datastore.NumberofBlocks();
for (int i = 0; i < c; i++) // go through blocks
{
if (visited.Get(i))
{
continue;
}
byte[] b = _datastore.ReadBlockBytes(i, _blockheader.Length + 255);
int bnum = Helper.ToInt32(b, 0);
if (bnum > 0) // check if a start block
{
visited.Set(i, true);
_datastore.FreeBlock(i); // mark as free
continue;
}
AllocationBlock ab = new AllocationBlock();
// start block found
int blocknumexpected = 0;
int next = ParseBlockHeader(ab, b, blocknumexpected);
int last = 0;
bool freelast = false;
AllocationBlock old = null;
if (keys.Get(ab.key, out last))
{
old = this.FillAllocationBlock(last);
freelast = true;
}
blocknumexpected++;
bool failed = false;
if (ab.deleteKey == false)
{
while (next > 0) // read the blocks
{
ab.Blocks.Add(next);
b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen);
next = ParseBlockHeader(ab, b, blocknumexpected);
if (next == -1) // non matching block
{
failed = true;
break;
}
blocknumexpected++;
}
}
else
{
failed = true;
keys.RemoveKey(ab.key);
}
// new data ok
if (failed == false)
{
keys.Set(ab.key, ab.blocknumber); // valid block found
if (freelast) // free the old blocks
{
_datastore.FreeBlocks(old.Blocks);
}
}
visited.Set(i, true);
}
// all ok delete temp.$ file
if (File.Exists(_Path + _dirtyFilename))
{
File.Delete(_Path + _dirtyFilename);
}
}
catch (Exception ex)
{
_log.Error(ex);
}
finally
{
_log.Debug("Shutting down files and index");
_datastore.Shutdown();
keys.SaveIndex();
keys.Shutdown();
}
}
private WAHBitArray ExecutionPlan(string filter, int maxsize)
{
//_log.Debug("query : " + filter);
DateTime dt = FastDateTime.Now;
// query indexes
string[] words = filter.Split(' ');
//bool defaulttoand = true;
//if (filter.IndexOfAny(new char[] { '+', '-' }, 0) > 0)
// defaulttoand = false;
WAHBitArray found = null;// WAHBitArray.Fill(maxsize);
foreach (string s in words)
{
int c;
bool not = false;
string word = s;
if (s == "")
{
continue;
}
OPERATION op = OPERATION.AND;
//if (defaulttoand)
// op = OPERATION.AND;
if (word.StartsWith("+"))
{
op = OPERATION.OR;
word = s.Replace("+", "");
}
if (word.StartsWith("-"))
{
op = OPERATION.ANDNOT;
word = s.Replace("-", "");
not = true;
if (found == null) // leading with - -> "-oak hill"
{
found = WAHBitArray.Fill(maxsize);
}
}
if (word.Contains("*") || word.Contains("?"))
{
WAHBitArray wildbits = new WAHBitArray();
// do wildcard search
Regex reg = new Regex("^" + word.Replace("*", ".*").Replace("?", ".") + "$", RegexOptions.IgnoreCase);
foreach (string key in _words.Keys())
{
if (reg.IsMatch(key))
{
_words.TryGetValue(key, out c);
WAHBitArray ba = _bitmaps.GetBitmap(c);
wildbits = DoBitOperation(wildbits, ba, OPERATION.OR, maxsize);
}
}
if (found == null)
{
found = wildbits;
}
else
{
if (not) // "-oak -*l"
{
found = found.AndNot(wildbits);
}
else if (op == OPERATION.AND)
{
found = found.And(wildbits);
}
else
{
found = found.Or(wildbits);
}
}
}
else if (_words.TryGetValue(word.ToLowerInvariant(), out c))
{
// bits logic
WAHBitArray ba = _bitmaps.GetBitmap(c);
found = DoBitOperation(found, ba, op, maxsize);
}
else if (op == OPERATION.AND)
{
found = new WAHBitArray();
}
}
if (found == null)
{
return(new WAHBitArray());
}
// remove deleted docs
WAHBitArray ret;
if (_docMode)
{
ret = found.AndNot(_deleted.GetBits());
}
else
{
ret = found;
}
//_log.Debug("query time (ms) = " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
return(ret);
}
