private static WAHBitArray DoBitOperation(WAHBitArray bits, WAHBitArray c, OPERATION op, int maxsize) { if (bits != null) { switch (op) { case OPERATION.AND: bits = bits.And(c); break; case OPERATION.OR: bits = bits.Or(c); break; case OPERATION.ANDNOT: bits = bits.And(c.Not(maxsize)); break; } } else { bits = c; } return(bits); }
private WAHBitArray LoadBitmap(long offset) { var bc = new WAHBitArray(); if (offset == -1) { return(bc); } var ar = new List <uint>(); var type = WAHBitArray.TYPE.WAH; var bmp = _bitmapFileRead; { bmp.Seek(offset, SeekOrigin.Begin); var b = new byte[8]; bmp.Read(b, 0, 8); if (b[0] == (byte)'B' && b[1] == (byte)'M' && b[7] == 0) { type = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), b[6]); int c = Helper.ToInt32(b, 2); var buf = new byte[c * 4]; bmp.Read(buf, 0, c * 4); for (int i = 0; i < c; i++) { ar.Add((uint)Helper.ToInt32(buf, i * 4)); } } } bc = new WAHBitArray(type, ar.ToArray()); return(bc); }
private WAHBitArray internalGetBitmap(int recno)//, bool usecache) { lock (_readlock) { WAHBitArray ba = new WAHBitArray(); if (recno == -1) { return(ba); } if (_cache.TryGetValue(recno, out ba)) { return(ba); } else { long offset = 0; if (_offsetCache.TryGetValue(recno, out offset) == false) { byte[] b = new byte[8]; long off = ((long)recno) * 8; _recordFileRead.Seek(off, SeekOrigin.Begin); _recordFileRead.Read(b, 0, 8); offset = Helper.ToInt64(b, 0); _offsetCache.Add(recno, offset); } ba = LoadBitmap(offset); //if (usecache) _cache.Add(recno, ba); return(ba); } } }
private WAHBitArray internalGetBitmap(int recno) { lock (_readlock) { WAHBitArray ba = new WAHBitArray(); if (recno == -1) { return(ba); } if (_cache.TryGetValue(recno, out ba)) { return(ba); } else { long offset = 0; if (_offsetCache.TryGetValue(recno, out offset) == false) { offset = ReadRecordOffset(recno); _offsetCache.Add(recno, offset); } ba = LoadBitmap(offset); _cache.Add(recno, ba); return(ba); } } }
public void Commit(bool freeMemory) { if (_isDirty == false) { return; } using (new L(this)) { log.Debug("writing " + _FileName); int[] keys = _cache.Keys(); Array.Sort(keys); foreach (int k in keys) { WAHBitArray bmp = null; if (_cache.TryGetValue(k, out bmp) && bmp.isDirty) { SaveBitmap(k, bmp); bmp.FreeMemory(); bmp.isDirty = false; } } Flush(); if (freeMemory) { _cache = //new SafeDictionary<int, WAHBitArray>(); new SafeSortedList <int, WAHBitArray>(); log.Debug(" freeing cache"); } _isDirty = false; } }
//----------------------------------------------------------------- // BITMAP FILE FORMAT // 0 'B','M' // 2 uint count = 4 bytes // 6 Bitmap type : // 0 = int record list // 1 = uint bitmap // 2 = rec# indexes // 7 '0' // 8 uint data //----------------------------------------------------------------- private long SaveBitmapToFile(WAHBitArray bmp) { long off = _lastBitmapOffset; WAHBitArray.TYPE t; uint[] bits = bmp.GetCompressed(out t); byte[] b = new byte[bits.Length * 4 + 8]; // write header data b[0] = ((byte)'B'); b[1] = ((byte)'M'); Buffer.BlockCopy(Helper.GetBytes(bits.Length, false), 0, b, 2, 4); b[6] = (byte)t; b[7] = (byte)(0); for (int i = 0; i < bits.Length; i++) { byte[] u = Helper.GetBytes((int)bits[i], false); Buffer.BlockCopy(u, 0, b, i * 4 + 8, 4); } _bitmapFileWrite.Write(b, 0, b.Length); _lastBitmapOffset += b.Length; return(off); }
public IEnumerable <int> FindRows(string filter) { WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount()); // enumerate records return(bits.GetBitIndexes()); }
public void Commit(bool freeMemory) { if (_isDirty == false) { return; } using (new L(this)) { log.Debug("writing " + _FileName); int[] keys = _cache.Keys.OrderBy(x => x).ToArray(); foreach (int k in keys) { WAHBitArray bmp = null; if (_cache.TryGetValue(k, out bmp) && bmp.isDirty) { SaveBitmap(k, bmp); bmp.FreeMemory(); bmp.isDirty = false; } } Flush(); if (freeMemory) { //if (Global.UseLessMemoryStructures) // _cache = new SafeSortedList<int, WAHBitArray>(); //else _cache = new ConcurrentDictionary <int, WAHBitArray>(); log.Debug(" freeing cache"); } _isDirty = false; } }
private void Initialize(string filename, ushort blocksize) { if (File.Exists(filename) == false) { _datawrite = new FileStream(filename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite); } else { _datawrite = new FileStream(filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); } if (_datawrite.Length == 0) { CreateFileHeader(blocksize); // new file _datawrite.Write(_fileheader, 0, _fileheader.Length); _datawrite.Flush(); } else { ReadFileHeader(); _lastBlockNumber = (int)((_datawrite.Length - _fileheader.Length) / _BLOCKSIZE); _lastBlockNumber++; } _freeList = new WAHBitArray(); if (File.Exists(_Path + _filename + ".free")) { ReadFreeListBMPFile(_Path + _filename + ".free"); // delete file so if failure no big deal on restart File.Delete(_Path + _filename + ".free"); } }
public void SetDuplicate(int bitmaprecno, int record) { WAHBitArray ba = null; ba = GetBitmap(bitmaprecno); ba.Set(record, true); }
public VirtualScanner() { DetectedFragments = new ConcurrentDictionary<long, VAScanType>(); Artifacts = new ConcurrentDictionary<long, Extract>(); CheckMethods = new List<Func<long, long[], VAScanType>>(); ScanList = new WAHBitArray(); }
public void SetDuplicate(int bitmaprecno, int record) { using (new L(this)) { WAHBitArray ba = null; ba = internalGetBitmap(bitmaprecno); //GetBitmap(bitmaprecno); ba.Set(record, true); } }
private void doPageOperation(ref WAHBitArray res, int pageidx) { Page <T> page = LoadPage(_pageList.GetValue(pageidx).PageNumber); foreach (var k in page.tree.Keys) { int bn = page.tree[k].DuplicateBitmapNumber; res = res.Or(_index.GetDuplicateBitmap(bn)); } }
public WAHBitArray Xor(WAHBitArray op) { lock (_lock) { uint[] left; uint[] right; prelogic(op, out left, out right); for (int i = 0; i < left.Length; i++) { left[i] ^= right[i]; } return(new WAHBitArray(TYPE.Bitarray, left)); } }
public WAHBitArray Xor(WAHBitArray op) { this.CheckBitArray(); uint[] ints = op.GetUncompressed(); uint[] uncomp = this.GetUncompressed(); FixLengths(ints, uncomp); for (int i = 0; i < ints.Length; i++) { ints[i] ^= uncomp[i]; } return(new WAHBitArray(TYPE.Uncompressed_WAH, ints)); }
internal void Initialize() { if (_readfreeList != null) { _freeList = _readfreeList(); } else { _freeList = new WAHBitArray(); if (File.Exists(_Path + _filename + ".free")) { ReadFreeListBMPFile(_Path + _filename + ".free"); // delete file so if failure no big deal on restart File.Delete(_Path + _filename + ".free"); } } }
public IEnumerable <Document> FindDocuments(string filter) { WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount()); // enumerate documents foreach (int i in bits.GetBitIndexes()) { if (i > _lastDocNum - 1) { break; } string b = _docs.ReadData(i); Document d = fastJSON.JSON.ToObject <Document>(b); yield return(d); } }
public IEnumerable <string> FindDocumentFileNames(string filter) { WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount()); // enumerate documents foreach (int i in bits.GetBitIndexes()) { if (i > _lastDocNum - 1) { break; } string b = _docs.ReadData(i); var d = (Dictionary <string, object>)fastJSON.JSON.Parse(b); yield return(d["FileName"].ToString()); } }
private WAHBitArray doLessOp(RDBExpression exp, T key) { bool found = false; int pos = FindPageOrLowerPosition(key, ref found); WAHBitArray result = new WAHBitArray(); if (pos > 0) { // all the pages before for (int i = 0; i < pos - 1; i++) { doPageOperation(ref result, i); } } // key page Page <T> page = LoadPage(_pageList.GetValue(pos).PageNumber); T[] keys = page.tree.Keys(); Array.Sort(keys); // find better end position rather than last key pos = Array.BinarySearch <T>(keys, key); if (pos < 0) { pos = ~pos; } for (int i = 0; i <= pos; i++) { T k = keys[i]; if (k.CompareTo(key) > 0) { break; } int bn = page.tree[k].DuplicateBitmapNumber; if (k.CompareTo(key) < 0) { result = result.Or(_index.GetDuplicateBitmap(bn)); } if (exp == RDBExpression.LessEqual && k.CompareTo(key) == 0) { result = result.Or(_index.GetDuplicateBitmap(bn)); } } return(result); }
private void SaveBitmap(int recno, WAHBitArray bmp) { lock (_writelock) { long offset = SaveBitmapToFile(bmp); //long v; //if (_offsetCache.TryGetValue(recno, out v)) // _offsetCache[recno] = offset; //else // _offsetCache.Add(recno, offset); long pointer = ((long)recno) * 8; _recordFileWrite.Seek(pointer, SeekOrigin.Begin); byte[] b = new byte[8]; b = Helper.GetBytes(offset, false); _recordFileWrite.Write(b, 0, 8); } }
private WAHBitArray doMoreOp(RDBExpression exp, T key) { bool found = false; int pos = FindPageOrLowerPosition(key, ref found); WAHBitArray result = new WAHBitArray(); if (pos < _pageList.Count) { // all the pages after for (int i = pos + 1; i < _pageList.Count; i++) { doPageOperation(ref result, i); } } // key page Page <T> page = LoadPage(_pageList.GetValue(pos).PageNumber); T[] keys = page.tree.Keys(); Array.Sort(keys); // find better start position rather than 0 pos = Array.IndexOf <T>(keys, key); if (pos == -1) { pos = 0; } for (int i = pos; i < keys.Length; i++) { T k = keys[i]; int bn = page.tree[k].DuplicateBitmapNumber; if (k.CompareTo(key) > 0) { result = result.Or(_index.GetDuplicateBitmap(bn)); } if (exp == RDBExpression.GreaterEqual && k.CompareTo(key) == 0) { result = result.Or(_index.GetDuplicateBitmap(bn)); } } return(result); }
private void ReadFreeListBMPFile(string filename) { byte[] b = File.ReadAllBytes(filename); WAHBitArray.TYPE t = WAHBitArray.TYPE.WAH; int j = 0; if (b.Length % 4 > 0) // new format with the data type byte { t = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), b[0]); j = 1; } List <uint> ints = new List <uint>(); for (int i = 0; i < b.Length / 4; i++) { ints.Add((uint)Helper.ToInt32(b, (i * 4) + j)); } _freeList = new WAHBitArray(t, ints.ToArray()); }
private void ReadFile() { byte[] b = File.ReadAllBytes(_path + _filename); MemoryStream ms = new MemoryStream(b); BinaryReader br = new BinaryReader(ms); WAHBitArray.TYPE t = WAHBitArray.TYPE.WAH; if (b.Length % 4 > 0) // new format with the data type byte { byte tb = br.ReadByte(); t = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), tb); } List <uint> ints = new List <uint>(); for (int i = 0; i < b.Length / 4; i++) { ints.Add((uint)br.ReadInt32()); } _bits = new WAHBitArray(t, ints.ToArray()); }
public IEnumerable <T> FindDocuments <T>(string filter) { checkloaded(); WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount()); // enumerate documents foreach (int i in bits.GetBitIndexes()) { if (i > _lastDocNum - 1) { break; } string b = _docs.ReadData(i); T d = fastJSON.JSON.ToObject <T>(b, new fastJSON.JSONParameters { ParametricConstructorOverride = true }); yield return(d); } }
private void prelogic(WAHBitArray op, out uint[] left, out uint[] right) { this.CheckBitArray(); left = this.GetBitArray(); right = op.GetBitArray(); int ic = left.Length; int uc = right.Length; if (ic > uc) { uint[] ar = new uint[ic]; right.CopyTo(ar, 0); right = ar; } else if (ic < uc) { uint[] ar = new uint[uc]; left.CopyTo(ar, 0); left = ar; } }
private WAHBitArray LoadBitmap(long offset) { WAHBitArray bc = new WAHBitArray(); if (offset == -1) { return(bc); } List <uint> ar = new List <uint>(); WAHBitArray.TYPE type = WAHBitArray.TYPE.Compressed_WAH; FileStream bmp = _bitmapFileRead; { bmp.Seek(offset, SeekOrigin.Begin); byte[] b = new byte[8]; bmp.Read(b, 0, 8); if (b[0] == (byte)'B' && b[1] == (byte)'M' && b[7] == 0) { type = (b[6] == 0 ? WAHBitArray.TYPE.Indexes : WAHBitArray.TYPE.Compressed_WAH); int c = Helper.ToInt32(b, 2); byte[] buf = new byte[c * 4]; bmp.Read(buf, 0, c * 4); for (int i = 0; i < c; i++) { ar.Add((uint)Helper.ToInt32(buf, i * 4)); } } } bc = new WAHBitArray(type, ar.ToArray()); return(bc); }
private void RebuildDataFiles() { MGIndex<string> keys = null; try { // remove old free list if (File.Exists(_Path + "data.bmp")) File.Delete(_Path + "data.bmp"); _datastore = new StorageFileHF(_Path + "data.mghf", Global.HighFrequencyKVDiskBlockSize); _BlockSize = _datastore.GetBlockSize(); if (File.Exists(_Path + "keys.idx")) { _log.Debug("removing old keys index"); foreach (var f in Directory.GetFiles(_Path, "keys.*")) File.Delete(f); } keys = new MGIndex<string>(_Path, "keys.idx", 255, Global.PageItemCount, false); WAHBitArray visited = new WAHBitArray(); int c = _datastore.NumberofBlocks(); for (int i = 0; i < c; i++) // go through blocks { if (visited.Get(i)) continue; byte[] b = _datastore.ReadBlockBytes(i, _blockheader.Length + 255); int bnum = Helper.ToInt32(b, 0); if (bnum > 0) // check if a start block { visited.Set(i, true); _datastore.FreeBlock(i); // mark as free continue; } AllocationBlock ab = new AllocationBlock(); // start block found int blocknumexpected = 0; int next = ParseBlockHeader(ab, b, blocknumexpected); int last = 0; bool freelast = false; AllocationBlock old = null; if (keys.Get(ab.key, out last)) { old = this.FillAllocationBlock(last); freelast = true; } blocknumexpected++; bool failed = false; if (ab.deleteKey == false) { while (next > 0) // read the blocks { ab.Blocks.Add(next); b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen); next = ParseBlockHeader(ab, b, blocknumexpected); if (next == -1) // non matching block { failed = true; break; } blocknumexpected++; } } else { failed = true; keys.RemoveKey(ab.key); } // new data ok if (failed == false) { keys.Set(ab.key, ab.blocknumber);// valid block found if (freelast)// free the old blocks _datastore.FreeBlocks(old.Blocks); } visited.Set(i, true); } // all ok delete temp.$ file if (File.Exists(_Path + _dirtyFilename)) File.Delete(_Path + _dirtyFilename); } catch (Exception ex) { _log.Error(ex); } finally { _log.Debug("Shutting down files and index"); _datastore.Shutdown(); keys.SaveIndex(); keys.Shutdown(); } }
//----------------------------------------------------------------- // BITMAP FILE FORMAT // 0 'B','M' // 2 uint count = 4 bytes // 6 Bitmap type : // 0 = int record list // 1 = uint bitmap // 2 = rec# indexes // 7 '0' // 8 uint data //----------------------------------------------------------------- private long SaveBitmapToFile(WAHBitArray bmp) { long off = _lastBitmapOffset; WAHBitArray.TYPE t; uint[] bits = bmp.GetCompressed(out t); byte[] b = new byte[bits.Length * 4 + 8]; // write header data b[0] = ((byte)'B'); b[1] = ((byte)'M'); Buffer.BlockCopy(Helper.GetBytes(bits.Length, false), 0, b, 2, 4); b[6] = (byte)t; b[7] = (byte)(0); for (int i = 0; i < bits.Length; i++) { byte[] u = Helper.GetBytes((int)bits[i], false); Buffer.BlockCopy(u, 0, b, i * 4 + 8, 4); } _bitmapFileWrite.Write(b, 0, b.Length); _lastBitmapOffset += b.Length; return off; }
private void SaveBitmap(int recno, WAHBitArray bmp) { lock (_writelock) { long offset = SaveBitmapToFile(bmp); long v; if (_offsetCache.TryGetValue(recno, out v)) _offsetCache[recno] = offset; else _offsetCache.Add(recno, offset); long pointer = ((long)recno) * 8; _recordFileWrite.Seek(pointer, SeekOrigin.Begin); byte[] b = new byte[8]; b = Helper.GetBytes(offset, false); _recordFileWrite.Write(b, 0, 8); } }
private WAHBitArray LoadBitmap(long offset) { WAHBitArray bc = new WAHBitArray(); if (offset == -1) return bc; List<uint> ar = new List<uint>(); WAHBitArray.TYPE type = WAHBitArray.TYPE.WAH; FileStream bmp = _bitmapFileRead; { bmp.Seek(offset, SeekOrigin.Begin); byte[] b = new byte[8]; bmp.Read(b, 0, 8); if (b[0] == (byte)'B' && b[1] == (byte)'M' && b[7] == 0) { type = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), b[6]); int c = Helper.ToInt32(b, 2); byte[] buf = new byte[c * 4]; bmp.Read(buf, 0, c * 4); for (int i = 0; i < c; i++) { ar.Add((uint)Helper.ToInt32(buf, i * 4)); } } } bc = new WAHBitArray(type, ar.ToArray()); return bc; }
private WAHBitArray internalGetBitmap(int recno) { lock (_readlock) { WAHBitArray ba = new WAHBitArray(); if (recno == -1) return ba; if (_cache.TryGetValue(recno, out ba)) { return ba; } else { long offset = 0; if (_offsetCache.TryGetValue(recno, out offset) == false) { offset = ReadRecordOffset(recno); _offsetCache.Add(recno, offset); } ba = LoadBitmap(offset); _cache.Add(recno, ba); return ba; } } }
private static WAHBitArray DoBitOperation(WAHBitArray bits, WAHBitArray c, OPERATION op, int maxsize) { if (bits != null) { switch (op) { case OPERATION.AND: bits = bits.And(c); break; case OPERATION.OR: bits = bits.Or(c); break; case OPERATION.ANDNOT: bits = bits.And(c.Not(maxsize)); break; } } else bits = c; return bits; }
public void InPlaceOR(WAHBitArray left) { lock (_lock) _bits = _bits.Or(left); }
private WAHBitArray ExecutionPlan(string filter, int maxsize) { //_log.Debug("query : " + filter); DateTime dt = FastDateTime.Now; // query indexes string[] words = filter.Split(' '); //bool defaulttoand = true; //if (filter.IndexOfAny(new char[] { '+', '-' }, 0) > 0) // defaulttoand = false; WAHBitArray found = null;// WAHBitArray.Fill(maxsize); foreach (string s in words) { int c; bool not = false; string word = s; if (s == "") { continue; } OPERATION op = OPERATION.AND; //if (defaulttoand) // op = OPERATION.AND; if (word.StartsWith("+")) { op = OPERATION.OR; word = s.Replace("+", ""); } if (word.StartsWith("-")) { op = OPERATION.ANDNOT; word = s.Replace("-", ""); not = true; if (found == null) // leading with - -> "-oak hill" { found = WAHBitArray.Fill(maxsize); } } if (word.Contains("*") || word.Contains("?")) { WAHBitArray wildbits = new WAHBitArray(); // do wildcard search Regex reg = new Regex("^" + word.Replace("*", ".*").Replace("?", ".") + "$", RegexOptions.IgnoreCase); foreach (string key in _words.Keys()) { if (reg.IsMatch(key)) { _words.TryGetValue(key, out c); WAHBitArray ba = _bitmaps.GetBitmap(c); wildbits = DoBitOperation(wildbits, ba, OPERATION.OR, maxsize); } } if (found == null) { found = wildbits; } else { if (not) // "-oak -*l" { found = found.AndNot(wildbits); } else if (op == OPERATION.AND) { found = found.And(wildbits); } else { found = found.Or(wildbits); } } } else if (_words.TryGetValue(word.ToLowerInvariant(), out c)) { // bits logic WAHBitArray ba = _bitmaps.GetBitmap(c); found = DoBitOperation(found, ba, op, maxsize); } else if (op == OPERATION.AND) { found = new WAHBitArray(); } } if (found == null) { return(new WAHBitArray()); } // remove deleted docs WAHBitArray ret; if (_docMode) { ret = found.AndNot(_deleted.GetBits()); } else { ret = found; } //_log.Debug("query time (ms) = " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); return(ret); }
public void InPlaceOR(WAHBitArray left) { _bits = _bits.Or(left); }
private void ReadFile() { byte[] b = File.ReadAllBytes(_path + _filename); WAHBitArray.TYPE t = WAHBitArray.TYPE.WAH; int j = 0; if (b.Length % 4 > 0) // new format with the data type byte { t = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), b[0]); j = 1; } List<uint> ints = new List<uint>(); for (int i = 0; i < b.Length / 4; i++) { ints.Add((uint)Helper.ToInt32(b, (i * 4) + j)); } _bits = new WAHBitArray(t, ints.ToArray()); }
private WAHBitArray ExecutionPlan(string filter, int maxsize) { //_log.Debug("query : " + filter); DateTime dt = FastDateTime.Now; // query indexes string[] words = filter.Split(' '); //bool defaulttoand = true; //if (filter.IndexOfAny(new char[] { '+', '-' }, 0) > 0) // defaulttoand = false; WAHBitArray found = null;// WAHBitArray.Fill(maxsize); foreach (string s in words) { int c; bool not = false; string word = s; if (s == "") continue; OPERATION op = OPERATION.AND; //if (defaulttoand) // op = OPERATION.AND; if (word.StartsWith("+")) { op = OPERATION.OR; word = s.Replace("+", ""); } if (word.StartsWith("-")) { op = OPERATION.ANDNOT; word = s.Replace("-", ""); not = true; if (found == null) // leading with - -> "-oak hill" { found = WAHBitArray.Fill(maxsize); } } if (word.Contains("*") || word.Contains("?")) { WAHBitArray wildbits = new WAHBitArray(); // do wildcard search Regex reg = new Regex("^" + word.Replace("*", ".*").Replace("?", ".") + "$", RegexOptions.IgnoreCase); foreach (string key in _words.Keys()) { if (reg.IsMatch(key)) { _words.TryGetValue(key, out c); WAHBitArray ba = _bitmaps.GetBitmap(c); wildbits = DoBitOperation(wildbits, ba, OPERATION.OR, maxsize); } } if (found == null) found = wildbits; else { if (not) // "-oak -*l" found = found.AndNot(wildbits); else if (op == OPERATION.AND) found = found.And(wildbits); else found = found.Or(wildbits); } } else if (_words.TryGetValue(word.ToLowerInvariant(), out c)) { // bits logic WAHBitArray ba = _bitmaps.GetBitmap(c); found = DoBitOperation(found, ba, op, maxsize); } else if (op == OPERATION.AND) found = null; } if (found == null) return new WAHBitArray(); // remove deleted docs WAHBitArray ret; if (_docMode) ret = found.AndNot(_deleted.GetBits()); else ret = found; //_log.Debug("query time (ms) = " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); return ret; }
public Mem(String mFile, uint[] BitmapArray = null, MemoryDescriptor Override = null) { GapScanSize = 0x10000000; StartOfMemory = Override != null ? Override.StartOfMemmory : 0; MapViewBase = 0; MapViewSize = MapWindowSize = (0x1000 * 0x1000 * 4); // Due to the physical page allocation algorithm a modest window is probably fine // so not even 1/2 the size of the window which was only getting < 50% hit ratio at best // PageCache may be better off than a huge window... if (PageCache == null) PageCache = new ConcurrentDictionary<long, long[]>(8, PageCacheMax); if (BitmapArray != null) pfnTableIdx = new WAHBitArray(WAHBitArray.TYPE.Bitarray, BitmapArray); else pfnTableIdx = new WAHBitArray(); // 32bit's of pages should be plenty? pfnTableIdx.Length = (int) (MapViewSize / 0x1000); if (File.Exists(mFile)) { MemoryDump = mFile; FileSize = new FileInfo(MemoryDump).Length; if (Override != null) MD = Override; else { MD = new MemoryDescriptor(FileSize); if (DetectedDescriptor != null) MD = DetectedDescriptor; } MapViewSize = FileSize < MapWindowSize ? FileSize : MapWindowSize; var lmindex = Interlocked.Increment(ref mindex); var mapName = Path.GetFileNameWithoutExtension(MemoryDump) + lmindex.ToString(); mapStream = new FileStream(MemoryDump, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); mappedFile = MemoryMappedFile.CreateFromFile(mapStream, mapName, 0, MemoryMappedFileAccess.Read, null, HandleInheritability.Inheritable, false); mappedAccess = mappedFile.CreateViewAccessor( MapViewBase, MapViewSize, MemoryMappedFileAccess.Read); DiscoveredGaps = new Dictionary<long, long>(); } }
// TODO: Figure out if MemoryCopy or BlockCopy ... public string WriteRange(VIRTUAL_ADDRESS KEY, PFN VALUE, string BaseFileName, Mem PhysMemReader = null, bool SinglePFNStore = false, bool DumpNULL = false) { if (SinglePFNStore && SISmap == null) SISmap = new WAHBitArray(); if(SinglePFNStore) { if (SISmap.Get((int)VALUE.PTE.PFN)) return string.Empty; SISmap.Set((int)VALUE.PTE.PFN, true); } bool canAppend = false; var saveLoc = BaseFileName + KEY.Address.ToString("X") + ".bin"; var lastLoc = BaseFileName + (KEY.Address - ContigSize).ToString("X") + ".bin"; if (File.Exists(lastLoc)) { canAppend = true; ContigSize += 0x1000; saveLoc = lastLoc; } else ContigSize = 0x1000; //unsafe //{ var block = new long[0x200]; // 0x200 * 8 = 4k var bpage = new byte[0x1000]; //fixed (void* lp = block, bp = bpage) //{ if (DiagOutput) WriteColor(VALUE.PTE.Valid ? ConsoleColor.Cyan : ConsoleColor.Red, $"VA: {KEY:X12} \t PFN: {VALUE.PTE}"); // if we have invalid (software managed) page table entries // the data may be present, or a prototype or actually in swap. // for the moment were only going to dump hardware managed data // or feel free to patch this up ;) if (!VALUE.PTE.Valid) return string.Empty; if (VALUE.PTE.LargePage) { bool GoodRead = false; using (var lsavefile = File.OpenWrite(saveLoc)) { // 0x200 * 4kb = 2MB // TODO: Large pages properly? // TODO: PageCache is still broken in some cases... disable for now here for (int i = 0; i < 0x200; i++) { PhysMemReader.GetPageForPhysAddr(VALUE.PTE, ref block, ref GoodRead); VALUE.PTE.PTE += 0x1000; if(!GoodRead) block = new long[0x200]; Buffer.BlockCopy(block, 0, bpage, 0, 4096); //Buffer.MemoryCopy(lp, bp, 4096, 4096); lsavefile.Write(bpage, 0, 4096); //lsavefile.Write(bpage, 0, 4096); } return lastLoc; } } else { try { PhysMemReader.GetPageForPhysAddr(VALUE.PTE, ref block); } catch (Exception ex) { } if (block != null) { if (DumpNULL || !UnsafeHelp.IsZero(block)) { Buffer.BlockCopy(block, 0, bpage, 0, 4096); //Buffer.MemoryCopy(lp, bp, 4096, 4096); using (var savefile = (canAppend ? File.Open(lastLoc, FileMode.Append, FileAccess.Write, FileShare.ReadWrite) : File.OpenWrite(saveLoc))) savefile.Write(bpage, 0, 4096); //savefile.Write(bpage, 0, 4096); return lastLoc; } } } ContigSize = 0; return string.Empty; //} //} }
private void Initialize(string filename, ushort blocksize) { if (File.Exists(filename) == false) _datawrite = new FileStream(filename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite); else _datawrite = new FileStream(filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); if (_datawrite.Length == 0) { CreateFileHeader(blocksize); // new file _datawrite.Write(_fileheader, 0, _fileheader.Length); _datawrite.Flush(); } else { ReadFileHeader(); _lastBlockNumber = (int)((_datawrite.Length - _fileheader.Length) / _BLOCKSIZE); _lastBlockNumber++; } _freeList = new WAHBitArray(); if (File.Exists(_Path + _filename + ".free")) { ReadFreeListBMPFile(_Path + _filename + ".free"); // delete file so if failure no big deal on restart File.Delete(_Path + _filename + ".free"); } }
public WAHBitArray Xor(WAHBitArray op) { lock (_lock) { uint[] left; uint[] right; prelogic(op, out left, out right); for (int i = 0; i < left.Length; i++) left[i] ^= right[i]; return new WAHBitArray(TYPE.Bitarray, left); } }
public WAHBitArray Query(RDBExpression ex, object from, int maxsize) { // always return everything return(WAHBitArray.Fill(maxsize)); }
public WAHBitArray Query(object fromkey, object tokey, int maxsize) { return(WAHBitArray.Fill(maxsize)); // TODO : all or none?? }
public static Mem InitMem(String mFile, AMemoryRunDetector Detector, uint[] BitmapArray = null) //: this() { var thiz = new Mem(); thiz.StartOfMemory = Detector != null ? Detector.StartOfMem : 0; if (Detector != null) { thiz.StartOfMemory = Detector.StartOfMem; thiz.MD = Detector; } #if USE_BITMAP // maybe there's a bit map we can use from a DMP file if (BitmapArray != null) pfnTableIdx = new WAHBitArray(WAHBitArray.TYPE.Bitarray, BitmapArray); else pfnTableIdx = new WAHBitArray(); // 32bit's of pages should be plenty? pfnTableIdx.Length = (int) (MapViewSize / 0x1000); #endif if (File.Exists(mFile)) { thiz.MemoryDump = mFile; thiz.FileSize = new FileInfo(mFile).Length; if (Detector != null) thiz.MD = Detector; } thiz.SetupStreams(); return thiz; }