/// <summary> /// 内部获取位图 /// </summary> /// <param name="recno"></param> /// <returns></returns> private WAHBitArray internalGetBitmap(int recno) { lock (_readlock) { WAHBitArray ba = new WAHBitArray(); if (recno == -1)//-1等于不存在,立即创建返回 { return(ba); } if (_cache.TryGetValue(recno, out ba)) { return(ba); } else { long offset = 0; //if (_offsetCache.TryGetValue(recno, out offset) == false) { offset = ReadRecordOffset(recno); // _offsetCache.Add(recno, offset); } ba = LoadBitmap(offset); _cache.Add(recno, ba); return(ba); } } }
private static WAHBitArray DoBitOperation(WAHBitArray bits, WAHBitArray c, OPERATION op, int maxsize) { if (bits != null) { switch (op) { case OPERATION.AND: bits = bits.And(c); break; case OPERATION.OR: bits = bits.Or(c); break; case OPERATION.ANDNOT: bits = bits.And(c.Not(maxsize)); break; } } else { bits = c; } return(bits); }
/// <summary> /// /// </summary> /// <param name="freeMemory">是否释放内存</param> public void Commit(bool freeMemory) { if (_isDirty == false) { return; } using (new L(this)) { log.Debug("writing " + _FileName); int[] keys = _cache.Keys(); Array.Sort(keys); foreach (int k in keys) { WAHBitArray bmp = null; if (_cache.TryGetValue(k, out bmp) && bmp.isDirty) { this.SaveBitmap(k, bmp); bmp.FreeMemory(); bmp.isDirty = false; } } Flush(); if (freeMemory) { _cache = //new SafeDictionary<int, WAHBitArray>(); new SafeSortedList <int, WAHBitArray>(); log.Debug(" freeing cache"); } _isDirty = false; } }
/// <summary> /// 保存到文件 /// ----------------------------------------------------------------- /// BITMAP FILE FORMAT /// 0 'B','M' /// 2 uint count = 4 bytes /// 6 Bitmap type : /// 0 = int record list /// 1 = uint bitmap /// 2 = rec# indexes /// 7 '0' /// 8 uint data /// ----------------------------------------------------------------- /// </summary> /// <param name="bmp"></param> /// <returns>文件中的位置</returns> private long SaveBitmapToFile(WAHBitArray bmp) { long off = _lastBitmapOffset; WAHBitArray.TYPE t; uint[] bits = bmp.GetCompressed(out t); byte[] b = new byte[bits.Length * 4 + 8]; // write header data b[0] = ((byte)'B'); b[1] = ((byte)'M'); Buffer.BlockCopy(Helper.GetBytes(bits.Length, false), 0, b, 2, 4); b[6] = (byte)t; b[7] = (byte)(0); for (int i = 0; i < bits.Length; i++) { byte[] u = Helper.GetBytes((int)bits[i], false); Buffer.BlockCopy(u, 0, b, i * 4 + 8, 4); } _bitmapFileWrite.Write(b, 0, b.Length); _lastBitmapOffset += b.Length; return(off); }
public IEnumerable <int> FindRows(string filter) { checkloaded(); WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount()); // enumerate records return(bits.GetBitIndexes()); }
/// <summary> /// /// </summary> /// <param name="bitmaprecno">索引编号</param> /// <param name="record">文档编号</param> public void SetDuplicate(int bitmaprecno, int record) { using (new L(this)) { WAHBitArray ba = null; ba = internalGetBitmap(bitmaprecno); //GetBitmap(bitmaprecno); ba.Set(record, true); _isDirty = true; } }
private void doPageOperation(ref WAHBitArray res, int pageidx) { Page <T> page = LoadPage(_pageList.GetValue(pageidx).PageNumber); T[] keys = page.tree.Keys(); // avoid sync issues foreach (var k in keys) { int bn = page.tree[k].DuplicateBitmapNumber; res = res.Or(_index.GetDuplicateBitmap(bn)); } }
public WAHBitArray Xor(WAHBitArray op) { lock (_lock) { uint[] left; uint[] right; prelogic(op, out left, out right); for (int i = 0; i < left.Length; i++) { left[i] ^= right[i]; } return(new WAHBitArray(TYPE.Bitarray, left)); } }
internal void Initialize() { if (_readfreeList != null) { _freeList = _readfreeList(); } else { _freeList = new WAHBitArray(); if (File.Exists(_Path + _filename + ".free")) { ReadFreeListBMPFile(_Path + _filename + ".free"); // delete file so if failure no big deal on restart File.Delete(_Path + _filename + ".free"); } } }
private void SaveBitmap(int recno, WAHBitArray bmp) { lock (_writelock) { long offset = SaveBitmapToFile(bmp); //long v; //if (_offsetCache.TryGetValue(recno, out v)) // _offsetCache[recno] = offset; //else // _offsetCache.Add(recno, offset); long pointer = ((long)recno) * 8; _recordFileWrite.Seek(pointer, SeekOrigin.Begin); byte[] b = new byte[8]; b = Helper.GetBytes(offset, false); _recordFileWrite.Write(b, 0, 8); } }
private WAHBitArray doMoreOp(RDBExpression exp, T key) { bool found = false; int pos = FindPageOrLowerPosition(key, ref found); WAHBitArray result = new WAHBitArray(); if (pos < _pageList.Count) { // all the pages after for (int i = pos + 1; i < _pageList.Count; i++) { doPageOperation(ref result, i); } } // key page Page <T> page = LoadPage(_pageList.GetValue(pos).PageNumber); T[] keys = page.tree.Keys(); Array.Sort(keys); // find better start position rather than 0 pos = Array.IndexOf <T>(keys, key); if (pos == -1) { pos = 0; } for (int i = pos; i < keys.Length; i++) { T k = keys[i]; int bn = page.tree[k].DuplicateBitmapNumber; if (k.CompareTo(key) > 0) { result = result.Or(_index.GetDuplicateBitmap(bn)); } if (exp == RDBExpression.GreaterEqual && k.CompareTo(key) == 0) { result = result.Or(_index.GetDuplicateBitmap(bn)); } } return(result); }
private void ReadFreeListBMPFile(string filename) { byte[] b = File.ReadAllBytes(filename); WAHBitArray.TYPE t = WAHBitArray.TYPE.WAH; int j = 0; if (b.Length % 4 > 0) // new format with the data type byte { t = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), b[0]); j = 1; } List <uint> ints = new List <uint>(); for (int i = 0; i < b.Length / 4; i++) { ints.Add((uint)Helper.ToInt32(b, (i * 4) + j)); } _freeList = new WAHBitArray(t, ints.ToArray()); }
public IEnumerable <string> FindDocumentFileNames(string filter) { checkloaded(); WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount()); // enumerate documents foreach (int i in bits.GetBitIndexes()) { if (i > _lastDocNum - 1) { break; } string b = _docs.ReadData(i); //var d = (Dictionary<string, object>)fastJSON.JSON.Parse(b);//fastJSON //yield return d["FileName"].ToString(); var d = _json.DeserializeObject <Document>(b); yield return(d.FileName); } }
public IEnumerable <T> FindDocuments <T>(string filter) { checkloaded(); WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount()); // enumerate documents foreach (int i in bits.GetBitIndexes()) { if (i > _lastDocNum - 1) { break; } string b = _docs.ReadData(i); //T d = fastJSON.JSON.ToObject<T>(b, new fastJSON.JSONParameters { ParametricConstructorOverride = true }); //fastJSON T d = _json.DeserializeObject <T>(b); yield return(d); } }
private void ReadFile() { byte[] b = File.ReadAllBytes(_path + _filename); MemoryStream ms = new MemoryStream(b); BinaryReader br = new BinaryReader(ms); WAHBitArray.TYPE t = WAHBitArray.TYPE.WAH; if (b.Length % 4 > 0) // new format with the data type byte { byte tb = br.ReadByte(); t = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), tb); } List <uint> ints = new List <uint>(); for (int i = 0; i < b.Length / 4; i++) { ints.Add((uint)br.ReadInt32()); } _bits = new WAHBitArray(t, ints.ToArray()); }
private WAHBitArray doLessOp(RDBExpression exp, T key) { bool found = false; int pos = FindPageOrLowerPosition(key, ref found); WAHBitArray result = new WAHBitArray(); if (pos > 0) { // all the pages before for (int i = 0; i < pos - 1; i++) { doPageOperation(ref result, i); } } // key page Page <T> page = LoadPage(_pageList.GetValue(pos).PageNumber); T[] keys = page.tree.Keys(); Array.Sort(keys); for (int i = 0; i < keys.Length; i++) { T k = keys[i]; if (k.CompareTo(key) > 0) { break; } int bn = page.tree[k].DuplicateBitmapNumber; if (k.CompareTo(key) < 0) { result = result.Or(_index.GetDuplicateBitmap(bn)); } if (exp == RDBExpression.LessEqual && k.CompareTo(key) == 0) { result = result.Or(_index.GetDuplicateBitmap(bn)); } } return(result); }
private void prelogic(WAHBitArray op, out uint[] left, out uint[] right) { this.CheckBitArray(); left = this.GetBitArray(); right = op.GetBitArray(); int ic = left.Length; int uc = right.Length; if (ic > uc) { uint[] ar = new uint[ic]; right.CopyTo(ar, 0); right = ar; } else if (ic < uc) { uint[] ar = new uint[uc]; left.CopyTo(ar, 0); left = ar; } }
/// <summary> /// 加载位图 /// </summary> /// <param name="offset"></param> /// <returns></returns> private WAHBitArray LoadBitmap(long offset) { WAHBitArray bc = new WAHBitArray(); if (offset == -1) { return(bc); } List <uint> ar = new List <uint>(); WAHBitArray.TYPE type = WAHBitArray.TYPE.WAH; FileStream bmp = _bitmapFileRead; { bmp.Seek(offset, SeekOrigin.Begin); byte[] b = new byte[8]; bmp.Read(b, 0, 8); //读取头部 if (b[0] == (byte)'B' && b[1] == (byte)'M' && b[7] == 0) //验证 { type = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), b[6]); int c = Helper.ToInt32(b, 2); byte[] buf = new byte[c * 4]; bmp.Read(buf, 0, c * 4); for (int i = 0; i < c; i++) { ar.Add((uint)Helper.ToInt32(buf, i * 4)); } } } bc = new WAHBitArray(type, ar.ToArray()); return(bc); }
public WAHBitArray Query(RDBExpression ex, object from, int maxsize) { // always return everything return(WAHBitArray.Fill(maxsize)); }
public void InPlaceOR(WAHBitArray left) { lock (_lock) _bits = _bits.Or(left); }
private void RebuildDataFiles() { MGIndex <string> keys = null; try { // remove old free list if (File.Exists(_Path + "data.bmp")) { File.Delete(_Path + "data.bmp"); } _datastore = new StorageFileHF(Path.Combine(_Path, "data.mghf"), Global.HighFrequencyKVDiskBlockSize); _BlockSize = _datastore.GetBlockSize(); if (File.Exists(_Path + "keys.idx")) { _log.Debug("removing old keys index"); foreach (var f in Directory.GetFiles(_Path, "keys.*")) { File.Delete(f); } } keys = new MGIndex <string>(_Path, "keys.idx", 255, /*Global.PageItemCount,*/ false); WAHBitArray visited = new WAHBitArray(); int c = _datastore.NumberofBlocks(); for (int i = 0; i < c; i++) // go through blocks { if (visited.Get(i)) { continue; } byte[] b = _datastore.ReadBlockBytes(i, _blockheader.Length + 255); int bnum = Helper.ToInt32(b, 0); if (bnum > 0) // check if a start block { visited.Set(i, true); _datastore.FreeBlock(i); // mark as free continue; } AllocationBlock ab = new AllocationBlock(); // start block found int blocknumexpected = 0; int next = ParseBlockHeader(ab, b, blocknumexpected); int last = 0; bool freelast = false; AllocationBlock old = null; if (keys.Get(ab.key, out last)) { old = this.FillAllocationBlock(last); freelast = true; } blocknumexpected++; bool failed = false; if (ab.deleteKey == false) { while (next > 0) // read the blocks { ab.Blocks.Add(next); b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen); next = ParseBlockHeader(ab, b, blocknumexpected); if (next == -1) // non matching block { failed = true; break; } blocknumexpected++; } } else { failed = true; keys.RemoveKey(ab.key); } // new data ok if (failed == false) { keys.Set(ab.key, ab.blocknumber); // valid block found if (freelast) // free the old blocks { _datastore.FreeBlocks(old.Blocks); } } visited.Set(i, true); } // all ok delete temp.$ file if (File.Exists(_Path + _dirtyFilename)) { File.Delete(_Path + _dirtyFilename); } } catch (Exception ex) { _log.Error(ex); } finally { _log.Debug("Shutting down files and index"); _datastore.Shutdown(); keys.SaveIndex(); keys.Shutdown(); } }
private WAHBitArray ExecutionPlan(string filter, int maxsize) { //_log.Debug("query : " + filter); DateTime dt = FastDateTime.Now; // query indexes string[] words = filter.Split(' '); //bool defaulttoand = true; //if (filter.IndexOfAny(new char[] { '+', '-' }, 0) > 0) // defaulttoand = false; WAHBitArray found = null;// WAHBitArray.Fill(maxsize); foreach (string s in words) { int c; bool not = false; string word = s; if (s == "") { continue; } OPERATION op = OPERATION.AND; //if (defaulttoand) // op = OPERATION.AND; if (word.StartsWith("+")) { op = OPERATION.OR; word = s.Replace("+", ""); } if (word.StartsWith("-")) { op = OPERATION.ANDNOT; word = s.Replace("-", ""); not = true; if (found == null) // leading with - -> "-oak hill" { found = WAHBitArray.Fill(maxsize); } } if (word.Contains("*") || word.Contains("?")) { WAHBitArray wildbits = new WAHBitArray(); // do wildcard search Regex reg = new Regex("^" + word.Replace("*", ".*").Replace("?", ".") + "$", RegexOptions.IgnoreCase); foreach (string key in _words.Keys()) { if (reg.IsMatch(key)) { _words.TryGetValue(key, out c); WAHBitArray ba = _bitmaps.GetBitmap(c); wildbits = DoBitOperation(wildbits, ba, OPERATION.OR, maxsize); } } if (found == null) { found = wildbits; } else { if (not) // "-oak -*l" { found = found.AndNot(wildbits); } else if (op == OPERATION.AND) { found = found.And(wildbits); } else { found = found.Or(wildbits); } } } else if (_words.TryGetValue(word.ToLowerInvariant(), out c)) { // bits logic WAHBitArray ba = _bitmaps.GetBitmap(c); found = DoBitOperation(found, ba, op, maxsize); } else if (op == OPERATION.AND) { found = new WAHBitArray(); } } if (found == null) { return(new WAHBitArray()); } // remove deleted docs WAHBitArray ret; if (_docMode) { ret = found.AndNot(_deleted.GetBits()); } else { ret = found; } //_log.Debug("query time (ms) = " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); return(ret); }