private static WAHBitArray DoBitOperation(WAHBitArray bits, WAHBitArray c, OPERATION op, int maxsize)
{
if (bits != null)
{
switch (op)
{
case OPERATION.AND:
bits = bits.And(c);
break;
case OPERATION.OR:
bits = bits.Or(c);
break;
case OPERATION.ANDNOT:
bits = bits.And(c.Not(maxsize));
break;
}
}
else
{
bits = c;
}
return(bits);
}
private WAHBitArray LoadBitmap(long offset)
{
var bc = new WAHBitArray();
if (offset == -1)
{
return(bc);
}
var ar = new List <uint>();
var type = WAHBitArray.TYPE.WAH;
var bmp = _bitmapFileRead;
{
bmp.Seek(offset, SeekOrigin.Begin);
var b = new byte[8];
bmp.Read(b, 0, 8);
if (b[0] == (byte)'B' && b[1] == (byte)'M' && b[7] == 0)
{
type = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), b[6]);
int c = Helper.ToInt32(b, 2);
var buf = new byte[c * 4];
bmp.Read(buf, 0, c * 4);
for (int i = 0; i < c; i++)
{
ar.Add((uint)Helper.ToInt32(buf, i * 4));
}
}
}
bc = new WAHBitArray(type, ar.ToArray());
return(bc);
}
private WAHBitArray internalGetBitmap(int recno)//, bool usecache)
{
lock (_readlock)
{
WAHBitArray ba = new WAHBitArray();
if (recno == -1)
{
return(ba);
}
if (_cache.TryGetValue(recno, out ba))
{
return(ba);
}
else
{
long offset = 0;
if (_offsetCache.TryGetValue(recno, out offset) == false)
{
byte[] b = new byte[8];
long off = ((long)recno) * 8;
_recordFileRead.Seek(off, SeekOrigin.Begin);
_recordFileRead.Read(b, 0, 8);
offset = Helper.ToInt64(b, 0);
_offsetCache.Add(recno, offset);
}
ba = LoadBitmap(offset);
//if (usecache)
_cache.Add(recno, ba);
return(ba);
}
}
}
private WAHBitArray internalGetBitmap(int recno)
{
lock (_readlock)
{
WAHBitArray ba = new WAHBitArray();
if (recno == -1)
{
return(ba);
}
if (_cache.TryGetValue(recno, out ba))
{
return(ba);
}
else
{
long offset = 0;
if (_offsetCache.TryGetValue(recno, out offset) == false)
{
offset = ReadRecordOffset(recno);
_offsetCache.Add(recno, offset);
}
ba = LoadBitmap(offset);
_cache.Add(recno, ba);
return(ba);
}
}
}
public void Commit(bool freeMemory)
{
if (_isDirty == false)
{
return;
}
using (new L(this))
{
log.Debug("writing " + _FileName);
int[] keys = _cache.Keys();
Array.Sort(keys);
foreach (int k in keys)
{
WAHBitArray bmp = null;
if (_cache.TryGetValue(k, out bmp) && bmp.isDirty)
{
SaveBitmap(k, bmp);
bmp.FreeMemory();
bmp.isDirty = false;
}
}
Flush();
if (freeMemory)
{
_cache = //new SafeDictionary<int, WAHBitArray>();
new SafeSortedList <int, WAHBitArray>();
log.Debug(" freeing cache");
}
_isDirty = false;
}
}
//-----------------------------------------------------------------
// BITMAP FILE FORMAT
// 0 'B','M'
// 2 uint count = 4 bytes
// 6 Bitmap type :
// 0 = int record list
// 1 = uint bitmap
// 2 = rec# indexes
// 7 '0'
// 8 uint data
//-----------------------------------------------------------------
private long SaveBitmapToFile(WAHBitArray bmp)
{
long off = _lastBitmapOffset;
WAHBitArray.TYPE t;
uint[] bits = bmp.GetCompressed(out t);
byte[] b = new byte[bits.Length * 4 + 8];
// write header data
b[0] = ((byte)'B');
b[1] = ((byte)'M');
Buffer.BlockCopy(Helper.GetBytes(bits.Length, false), 0, b, 2, 4);
b[6] = (byte)t;
b[7] = (byte)(0);
for (int i = 0; i < bits.Length; i++)
{
byte[] u = Helper.GetBytes((int)bits[i], false);
Buffer.BlockCopy(u, 0, b, i * 4 + 8, 4);
}
_bitmapFileWrite.Write(b, 0, b.Length);
_lastBitmapOffset += b.Length;
return(off);
}
public IEnumerable <int> FindRows(string filter)
{
WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount());
// enumerate records
return(bits.GetBitIndexes());
}
public void Commit(bool freeMemory)
{
if (_isDirty == false)
{
return;
}
using (new L(this))
{
log.Debug("writing " + _FileName);
int[] keys = _cache.Keys.OrderBy(x => x).ToArray();
foreach (int k in keys)
{
WAHBitArray bmp = null;
if (_cache.TryGetValue(k, out bmp) && bmp.isDirty)
{
SaveBitmap(k, bmp);
bmp.FreeMemory();
bmp.isDirty = false;
}
}
Flush();
if (freeMemory)
{
//if (Global.UseLessMemoryStructures)
// _cache = new SafeSortedList<int, WAHBitArray>();
//else
_cache = new ConcurrentDictionary <int, WAHBitArray>();
log.Debug(" freeing cache");
}
_isDirty = false;
}
}
private void Initialize(string filename, ushort blocksize)
{
if (File.Exists(filename) == false)
{
_datawrite = new FileStream(filename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite);
}
else
{
_datawrite = new FileStream(filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
}
if (_datawrite.Length == 0)
{
CreateFileHeader(blocksize);
// new file
_datawrite.Write(_fileheader, 0, _fileheader.Length);
_datawrite.Flush();
}
else
{
ReadFileHeader();
_lastBlockNumber = (int)((_datawrite.Length - _fileheader.Length) / _BLOCKSIZE);
_lastBlockNumber++;
}
_freeList = new WAHBitArray();
if (File.Exists(_Path + _filename + ".free"))
{
ReadFreeListBMPFile(_Path + _filename + ".free");
// delete file so if failure no big deal on restart
File.Delete(_Path + _filename + ".free");
}
}
public void SetDuplicate(int bitmaprecno, int record)
{
WAHBitArray ba = null;
ba = GetBitmap(bitmaprecno);
ba.Set(record, true);
}
public VirtualScanner()
{
DetectedFragments = new ConcurrentDictionary<long, VAScanType>();
Artifacts = new ConcurrentDictionary<long, Extract>();
CheckMethods = new List<Func<long, long[], VAScanType>>();
ScanList = new WAHBitArray();
}
public void SetDuplicate(int bitmaprecno, int record)
{
using (new L(this))
{
WAHBitArray ba = null;
ba = internalGetBitmap(bitmaprecno); //GetBitmap(bitmaprecno);
ba.Set(record, true);
}
}
private void doPageOperation(ref WAHBitArray res, int pageidx)
{
Page <T> page = LoadPage(_pageList.GetValue(pageidx).PageNumber);
foreach (var k in page.tree.Keys)
{
int bn = page.tree[k].DuplicateBitmapNumber;
res = res.Or(_index.GetDuplicateBitmap(bn));
}
}
public WAHBitArray Xor(WAHBitArray op)
{
lock (_lock)
{
uint[] left;
uint[] right;
prelogic(op, out left, out right);
for (int i = 0; i < left.Length; i++)
{
left[i] ^= right[i];
}
return(new WAHBitArray(TYPE.Bitarray, left));
}
}
public WAHBitArray Xor(WAHBitArray op)
{
this.CheckBitArray();
uint[] ints = op.GetUncompressed();
uint[] uncomp = this.GetUncompressed();
FixLengths(ints, uncomp);
for (int i = 0; i < ints.Length; i++)
{
ints[i] ^= uncomp[i];
}
return(new WAHBitArray(TYPE.Uncompressed_WAH, ints));
}
internal void Initialize()
{
if (_readfreeList != null)
{
_freeList = _readfreeList();
}
else
{
_freeList = new WAHBitArray();
if (File.Exists(_Path + _filename + ".free"))
{
ReadFreeListBMPFile(_Path + _filename + ".free");
// delete file so if failure no big deal on restart
File.Delete(_Path + _filename + ".free");
}
}
}
public IEnumerable <Document> FindDocuments(string filter)
{
WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount());
// enumerate documents
foreach (int i in bits.GetBitIndexes())
{
if (i > _lastDocNum - 1)
{
break;
}
string b = _docs.ReadData(i);
Document d = fastJSON.JSON.ToObject <Document>(b);
yield return(d);
}
}
public IEnumerable <string> FindDocumentFileNames(string filter)
{
WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount());
// enumerate documents
foreach (int i in bits.GetBitIndexes())
{
if (i > _lastDocNum - 1)
{
break;
}
string b = _docs.ReadData(i);
var d = (Dictionary <string, object>)fastJSON.JSON.Parse(b);
yield return(d["FileName"].ToString());
}
}
private WAHBitArray doLessOp(RDBExpression exp, T key)
{
bool found = false;
int pos = FindPageOrLowerPosition(key, ref found);
WAHBitArray result = new WAHBitArray();
if (pos > 0)
{
// all the pages before
for (int i = 0; i < pos - 1; i++)
{
doPageOperation(ref result, i);
}
}
// key page
Page <T> page = LoadPage(_pageList.GetValue(pos).PageNumber);
T[] keys = page.tree.Keys();
Array.Sort(keys);
// find better end position rather than last key
pos = Array.BinarySearch <T>(keys, key);
if (pos < 0)
{
pos = ~pos;
}
for (int i = 0; i <= pos; i++)
{
T k = keys[i];
if (k.CompareTo(key) > 0)
{
break;
}
int bn = page.tree[k].DuplicateBitmapNumber;
if (k.CompareTo(key) < 0)
{
result = result.Or(_index.GetDuplicateBitmap(bn));
}
if (exp == RDBExpression.LessEqual && k.CompareTo(key) == 0)
{
result = result.Or(_index.GetDuplicateBitmap(bn));
}
}
return(result);
}
private void SaveBitmap(int recno, WAHBitArray bmp)
{
lock (_writelock)
{
long offset = SaveBitmapToFile(bmp);
//long v;
//if (_offsetCache.TryGetValue(recno, out v))
// _offsetCache[recno] = offset;
//else
// _offsetCache.Add(recno, offset);
long pointer = ((long)recno) * 8;
_recordFileWrite.Seek(pointer, SeekOrigin.Begin);
byte[] b = new byte[8];
b = Helper.GetBytes(offset, false);
_recordFileWrite.Write(b, 0, 8);
}
}
private WAHBitArray doMoreOp(RDBExpression exp, T key)
{
bool found = false;
int pos = FindPageOrLowerPosition(key, ref found);
WAHBitArray result = new WAHBitArray();
if (pos < _pageList.Count)
{
// all the pages after
for (int i = pos + 1; i < _pageList.Count; i++)
{
doPageOperation(ref result, i);
}
}
// key page
Page <T> page = LoadPage(_pageList.GetValue(pos).PageNumber);
T[] keys = page.tree.Keys();
Array.Sort(keys);
// find better start position rather than 0
pos = Array.IndexOf <T>(keys, key);
if (pos == -1)
{
pos = 0;
}
for (int i = pos; i < keys.Length; i++)
{
T k = keys[i];
int bn = page.tree[k].DuplicateBitmapNumber;
if (k.CompareTo(key) > 0)
{
result = result.Or(_index.GetDuplicateBitmap(bn));
}
if (exp == RDBExpression.GreaterEqual && k.CompareTo(key) == 0)
{
result = result.Or(_index.GetDuplicateBitmap(bn));
}
}
return(result);
}
private void ReadFreeListBMPFile(string filename)
{
byte[] b = File.ReadAllBytes(filename);
WAHBitArray.TYPE t = WAHBitArray.TYPE.WAH;
int j = 0;
if (b.Length % 4 > 0) // new format with the data type byte
{
t = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), b[0]);
j = 1;
}
List <uint> ints = new List <uint>();
for (int i = 0; i < b.Length / 4; i++)
{
ints.Add((uint)Helper.ToInt32(b, (i * 4) + j));
}
_freeList = new WAHBitArray(t, ints.ToArray());
}
private void ReadFile()
{
byte[] b = File.ReadAllBytes(_path + _filename);
MemoryStream ms = new MemoryStream(b);
BinaryReader br = new BinaryReader(ms);
WAHBitArray.TYPE t = WAHBitArray.TYPE.WAH;
if (b.Length % 4 > 0) // new format with the data type byte
{
byte tb = br.ReadByte();
t = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), tb);
}
List <uint> ints = new List <uint>();
for (int i = 0; i < b.Length / 4; i++)
{
ints.Add((uint)br.ReadInt32());
}
_bits = new WAHBitArray(t, ints.ToArray());
}
public IEnumerable <T> FindDocuments <T>(string filter)
{
checkloaded();
WAHBitArray bits = ExecutionPlan(filter, _docs.RecordCount());
// enumerate documents
foreach (int i in bits.GetBitIndexes())
{
if (i > _lastDocNum - 1)
{
break;
}
string b = _docs.ReadData(i);
T d = fastJSON.JSON.ToObject <T>(b, new fastJSON.JSONParameters {
ParametricConstructorOverride = true
});
yield return(d);
}
}
private void prelogic(WAHBitArray op, out uint[] left, out uint[] right)
{
this.CheckBitArray();
left = this.GetBitArray();
right = op.GetBitArray();
int ic = left.Length;
int uc = right.Length;
if (ic > uc)
{
uint[] ar = new uint[ic];
right.CopyTo(ar, 0);
right = ar;
}
else if (ic < uc)
{
uint[] ar = new uint[uc];
left.CopyTo(ar, 0);
left = ar;
}
}
private WAHBitArray LoadBitmap(long offset)
{
WAHBitArray bc = new WAHBitArray();
if (offset == -1)
{
return(bc);
}
List <uint> ar = new List <uint>();
WAHBitArray.TYPE type = WAHBitArray.TYPE.Compressed_WAH;
FileStream bmp = _bitmapFileRead;
{
bmp.Seek(offset, SeekOrigin.Begin);
byte[] b = new byte[8];
bmp.Read(b, 0, 8);
if (b[0] == (byte)'B' && b[1] == (byte)'M' && b[7] == 0)
{
type = (b[6] == 0 ? WAHBitArray.TYPE.Indexes : WAHBitArray.TYPE.Compressed_WAH);
int c = Helper.ToInt32(b, 2);
byte[] buf = new byte[c * 4];
bmp.Read(buf, 0, c * 4);
for (int i = 0; i < c; i++)
{
ar.Add((uint)Helper.ToInt32(buf, i * 4));
}
}
}
bc = new WAHBitArray(type, ar.ToArray());
return(bc);
}
private void RebuildDataFiles()
{
MGIndex<string> keys = null;
try
{
// remove old free list
if (File.Exists(_Path + "data.bmp"))
File.Delete(_Path + "data.bmp");
_datastore = new StorageFileHF(_Path + "data.mghf", Global.HighFrequencyKVDiskBlockSize);
_BlockSize = _datastore.GetBlockSize();
if (File.Exists(_Path + "keys.idx"))
{
_log.Debug("removing old keys index");
foreach (var f in Directory.GetFiles(_Path, "keys.*"))
File.Delete(f);
}
keys = new MGIndex<string>(_Path, "keys.idx", 255, Global.PageItemCount, false);
WAHBitArray visited = new WAHBitArray();
int c = _datastore.NumberofBlocks();
for (int i = 0; i < c; i++) // go through blocks
{
if (visited.Get(i))
continue;
byte[] b = _datastore.ReadBlockBytes(i, _blockheader.Length + 255);
int bnum = Helper.ToInt32(b, 0);
if (bnum > 0) // check if a start block
{
visited.Set(i, true);
_datastore.FreeBlock(i); // mark as free
continue;
}
AllocationBlock ab = new AllocationBlock();
// start block found
int blocknumexpected = 0;
int next = ParseBlockHeader(ab, b, blocknumexpected);
int last = 0;
bool freelast = false;
AllocationBlock old = null;
if (keys.Get(ab.key, out last))
{
old = this.FillAllocationBlock(last);
freelast = true;
}
blocknumexpected++;
bool failed = false;
if (ab.deleteKey == false)
{
while (next > 0) // read the blocks
{
ab.Blocks.Add(next);
b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen);
next = ParseBlockHeader(ab, b, blocknumexpected);
if (next == -1) // non matching block
{
failed = true;
break;
}
blocknumexpected++;
}
}
else
{
failed = true;
keys.RemoveKey(ab.key);
}
// new data ok
if (failed == false)
{
keys.Set(ab.key, ab.blocknumber);// valid block found
if (freelast)// free the old blocks
_datastore.FreeBlocks(old.Blocks);
}
visited.Set(i, true);
}
// all ok delete temp.$ file
if (File.Exists(_Path + _dirtyFilename))
File.Delete(_Path + _dirtyFilename);
}
catch (Exception ex)
{
_log.Error(ex);
}
finally
{
_log.Debug("Shutting down files and index");
_datastore.Shutdown();
keys.SaveIndex();
keys.Shutdown();
}
}
//-----------------------------------------------------------------
// BITMAP FILE FORMAT
// 0 'B','M'
// 2 uint count = 4 bytes
// 6 Bitmap type :
// 0 = int record list
// 1 = uint bitmap
// 2 = rec# indexes
// 7 '0'
// 8 uint data
//-----------------------------------------------------------------
private long SaveBitmapToFile(WAHBitArray bmp)
{
long off = _lastBitmapOffset;
WAHBitArray.TYPE t;
uint[] bits = bmp.GetCompressed(out t);
byte[] b = new byte[bits.Length * 4 + 8];
// write header data
b[0] = ((byte)'B');
b[1] = ((byte)'M');
Buffer.BlockCopy(Helper.GetBytes(bits.Length, false), 0, b, 2, 4);
b[6] = (byte)t;
b[7] = (byte)(0);
for (int i = 0; i < bits.Length; i++)
{
byte[] u = Helper.GetBytes((int)bits[i], false);
Buffer.BlockCopy(u, 0, b, i * 4 + 8, 4);
}
_bitmapFileWrite.Write(b, 0, b.Length);
_lastBitmapOffset += b.Length;
return off;
}
private void SaveBitmap(int recno, WAHBitArray bmp)
{
lock (_writelock)
{
long offset = SaveBitmapToFile(bmp);
long v;
if (_offsetCache.TryGetValue(recno, out v))
_offsetCache[recno] = offset;
else
_offsetCache.Add(recno, offset);
long pointer = ((long)recno) * 8;
_recordFileWrite.Seek(pointer, SeekOrigin.Begin);
byte[] b = new byte[8];
b = Helper.GetBytes(offset, false);
_recordFileWrite.Write(b, 0, 8);
}
}
private WAHBitArray LoadBitmap(long offset)
{
WAHBitArray bc = new WAHBitArray();
if (offset == -1)
return bc;
List<uint> ar = new List<uint>();
WAHBitArray.TYPE type = WAHBitArray.TYPE.WAH;
FileStream bmp = _bitmapFileRead;
{
bmp.Seek(offset, SeekOrigin.Begin);
byte[] b = new byte[8];
bmp.Read(b, 0, 8);
if (b[0] == (byte)'B' && b[1] == (byte)'M' && b[7] == 0)
{
type = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), b[6]);
int c = Helper.ToInt32(b, 2);
byte[] buf = new byte[c * 4];
bmp.Read(buf, 0, c * 4);
for (int i = 0; i < c; i++)
{
ar.Add((uint)Helper.ToInt32(buf, i * 4));
}
}
}
bc = new WAHBitArray(type, ar.ToArray());
return bc;
}
private WAHBitArray internalGetBitmap(int recno)
{
lock (_readlock)
{
WAHBitArray ba = new WAHBitArray();
if (recno == -1)
return ba;
if (_cache.TryGetValue(recno, out ba))
{
return ba;
}
else
{
long offset = 0;
if (_offsetCache.TryGetValue(recno, out offset) == false)
{
offset = ReadRecordOffset(recno);
_offsetCache.Add(recno, offset);
}
ba = LoadBitmap(offset);
_cache.Add(recno, ba);
return ba;
}
}
}
private static WAHBitArray DoBitOperation(WAHBitArray bits, WAHBitArray c, OPERATION op, int maxsize)
{
if (bits != null)
{
switch (op)
{
case OPERATION.AND:
bits = bits.And(c);
break;
case OPERATION.OR:
bits = bits.Or(c);
break;
case OPERATION.ANDNOT:
bits = bits.And(c.Not(maxsize));
break;
}
}
else
bits = c;
return bits;
}
public void InPlaceOR(WAHBitArray left)
{
lock (_lock)
_bits = _bits.Or(left);
}
private WAHBitArray ExecutionPlan(string filter, int maxsize)
{
//_log.Debug("query : " + filter);
DateTime dt = FastDateTime.Now;
// query indexes
string[] words = filter.Split(' ');
//bool defaulttoand = true;
//if (filter.IndexOfAny(new char[] { '+', '-' }, 0) > 0)
// defaulttoand = false;
WAHBitArray found = null;// WAHBitArray.Fill(maxsize);
foreach (string s in words)
{
int c;
bool not = false;
string word = s;
if (s == "")
{
continue;
}
OPERATION op = OPERATION.AND;
//if (defaulttoand)
// op = OPERATION.AND;
if (word.StartsWith("+"))
{
op = OPERATION.OR;
word = s.Replace("+", "");
}
if (word.StartsWith("-"))
{
op = OPERATION.ANDNOT;
word = s.Replace("-", "");
not = true;
if (found == null) // leading with - -> "-oak hill"
{
found = WAHBitArray.Fill(maxsize);
}
}
if (word.Contains("*") || word.Contains("?"))
{
WAHBitArray wildbits = new WAHBitArray();
// do wildcard search
Regex reg = new Regex("^" + word.Replace("*", ".*").Replace("?", ".") + "$", RegexOptions.IgnoreCase);
foreach (string key in _words.Keys())
{
if (reg.IsMatch(key))
{
_words.TryGetValue(key, out c);
WAHBitArray ba = _bitmaps.GetBitmap(c);
wildbits = DoBitOperation(wildbits, ba, OPERATION.OR, maxsize);
}
}
if (found == null)
{
found = wildbits;
}
else
{
if (not) // "-oak -*l"
{
found = found.AndNot(wildbits);
}
else if (op == OPERATION.AND)
{
found = found.And(wildbits);
}
else
{
found = found.Or(wildbits);
}
}
}
else if (_words.TryGetValue(word.ToLowerInvariant(), out c))
{
// bits logic
WAHBitArray ba = _bitmaps.GetBitmap(c);
found = DoBitOperation(found, ba, op, maxsize);
}
else if (op == OPERATION.AND)
{
found = new WAHBitArray();
}
}
if (found == null)
{
return(new WAHBitArray());
}
// remove deleted docs
WAHBitArray ret;
if (_docMode)
{
ret = found.AndNot(_deleted.GetBits());
}
else
{
ret = found;
}
//_log.Debug("query time (ms) = " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
return(ret);
}
public void InPlaceOR(WAHBitArray left)
{
_bits = _bits.Or(left);
}
private void ReadFile()
{
byte[] b = File.ReadAllBytes(_path + _filename);
WAHBitArray.TYPE t = WAHBitArray.TYPE.WAH;
int j = 0;
if (b.Length % 4 > 0) // new format with the data type byte
{
t = (WAHBitArray.TYPE)Enum.ToObject(typeof(WAHBitArray.TYPE), b[0]);
j = 1;
}
List<uint> ints = new List<uint>();
for (int i = 0; i < b.Length / 4; i++)
{
ints.Add((uint)Helper.ToInt32(b, (i * 4) + j));
}
_bits = new WAHBitArray(t, ints.ToArray());
}
private WAHBitArray ExecutionPlan(string filter, int maxsize)
{
//_log.Debug("query : " + filter);
DateTime dt = FastDateTime.Now;
// query indexes
string[] words = filter.Split(' ');
//bool defaulttoand = true;
//if (filter.IndexOfAny(new char[] { '+', '-' }, 0) > 0)
// defaulttoand = false;
WAHBitArray found = null;// WAHBitArray.Fill(maxsize);
foreach (string s in words)
{
int c;
bool not = false;
string word = s;
if (s == "") continue;
OPERATION op = OPERATION.AND;
//if (defaulttoand)
// op = OPERATION.AND;
if (word.StartsWith("+"))
{
op = OPERATION.OR;
word = s.Replace("+", "");
}
if (word.StartsWith("-"))
{
op = OPERATION.ANDNOT;
word = s.Replace("-", "");
not = true;
if (found == null) // leading with - -> "-oak hill"
{
found = WAHBitArray.Fill(maxsize);
}
}
if (word.Contains("*") || word.Contains("?"))
{
WAHBitArray wildbits = new WAHBitArray();
// do wildcard search
Regex reg = new Regex("^" + word.Replace("*", ".*").Replace("?", ".") + "$", RegexOptions.IgnoreCase);
foreach (string key in _words.Keys())
{
if (reg.IsMatch(key))
{
_words.TryGetValue(key, out c);
WAHBitArray ba = _bitmaps.GetBitmap(c);
wildbits = DoBitOperation(wildbits, ba, OPERATION.OR, maxsize);
}
}
if (found == null)
found = wildbits;
else
{
if (not) // "-oak -*l"
found = found.AndNot(wildbits);
else if (op == OPERATION.AND)
found = found.And(wildbits);
else
found = found.Or(wildbits);
}
}
else if (_words.TryGetValue(word.ToLowerInvariant(), out c))
{
// bits logic
WAHBitArray ba = _bitmaps.GetBitmap(c);
found = DoBitOperation(found, ba, op, maxsize);
}
else if (op == OPERATION.AND)
found = null;
}
if (found == null)
return new WAHBitArray();
// remove deleted docs
WAHBitArray ret;
if (_docMode)
ret = found.AndNot(_deleted.GetBits());
else
ret = found;
//_log.Debug("query time (ms) = " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
return ret;
}
public Mem(String mFile, uint[] BitmapArray = null, MemoryDescriptor Override = null)
{
GapScanSize = 0x10000000;
StartOfMemory = Override != null ? Override.StartOfMemmory : 0;
MapViewBase = 0;
MapViewSize = MapWindowSize = (0x1000 * 0x1000 * 4); // Due to the physical page allocation algorithm a modest window is probably fine
// so not even 1/2 the size of the window which was only getting < 50% hit ratio at best
// PageCache may be better off than a huge window...
if (PageCache == null)
PageCache = new ConcurrentDictionary<long, long[]>(8, PageCacheMax);
if (BitmapArray != null)
pfnTableIdx = new WAHBitArray(WAHBitArray.TYPE.Bitarray, BitmapArray);
else
pfnTableIdx = new WAHBitArray();
// 32bit's of pages should be plenty?
pfnTableIdx.Length = (int) (MapViewSize / 0x1000);
if (File.Exists(mFile))
{
MemoryDump = mFile;
FileSize = new FileInfo(MemoryDump).Length;
if (Override != null)
MD = Override;
else {
MD = new MemoryDescriptor(FileSize);
if (DetectedDescriptor != null)
MD = DetectedDescriptor;
}
MapViewSize = FileSize < MapWindowSize ? FileSize : MapWindowSize;
var lmindex = Interlocked.Increment(ref mindex);
var mapName = Path.GetFileNameWithoutExtension(MemoryDump) + lmindex.ToString();
mapStream = new FileStream(MemoryDump, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
mappedFile = MemoryMappedFile.CreateFromFile(mapStream,
mapName,
0,
MemoryMappedFileAccess.Read,
null,
HandleInheritability.Inheritable,
false);
mappedAccess = mappedFile.CreateViewAccessor(
MapViewBase,
MapViewSize,
MemoryMappedFileAccess.Read);
DiscoveredGaps = new Dictionary<long, long>();
}
}
// TODO: Figure out if MemoryCopy or BlockCopy ...
public string WriteRange(VIRTUAL_ADDRESS KEY, PFN VALUE, string BaseFileName, Mem PhysMemReader = null, bool SinglePFNStore = false, bool DumpNULL = false)
{
if (SinglePFNStore && SISmap == null)
SISmap = new WAHBitArray();
if(SinglePFNStore)
{
if (SISmap.Get((int)VALUE.PTE.PFN))
return string.Empty;
SISmap.Set((int)VALUE.PTE.PFN, true);
}
bool canAppend = false;
var saveLoc = BaseFileName + KEY.Address.ToString("X") + ".bin";
var lastLoc = BaseFileName + (KEY.Address - ContigSize).ToString("X") + ".bin";
if (File.Exists(lastLoc))
{
canAppend = true;
ContigSize += 0x1000;
saveLoc = lastLoc;
}
else
ContigSize = 0x1000;
//unsafe
//{
var block = new long[0x200]; // 0x200 * 8 = 4k
var bpage = new byte[0x1000];
//fixed (void* lp = block, bp = bpage)
//{
if (DiagOutput)
WriteColor(VALUE.PTE.Valid ? ConsoleColor.Cyan : ConsoleColor.Red, $"VA: {KEY:X12} \t PFN: {VALUE.PTE}");
// if we have invalid (software managed) page table entries
// the data may be present, or a prototype or actually in swap.
// for the moment were only going to dump hardware managed data
// or feel free to patch this up ;)
if (!VALUE.PTE.Valid)
return string.Empty;
if (VALUE.PTE.LargePage)
{
bool GoodRead = false;
using (var lsavefile = File.OpenWrite(saveLoc))
{
// 0x200 * 4kb = 2MB
// TODO: Large pages properly?
// TODO: PageCache is still broken in some cases... disable for now here
for (int i = 0; i < 0x200; i++)
{
PhysMemReader.GetPageForPhysAddr(VALUE.PTE, ref block, ref GoodRead);
VALUE.PTE.PTE += 0x1000;
if(!GoodRead)
block = new long[0x200];
Buffer.BlockCopy(block, 0, bpage, 0, 4096);
//Buffer.MemoryCopy(lp, bp, 4096, 4096);
lsavefile.Write(bpage, 0, 4096);
//lsavefile.Write(bpage, 0, 4096);
}
return lastLoc;
}
}
else
{
try { PhysMemReader.GetPageForPhysAddr(VALUE.PTE, ref block); } catch (Exception ex) { }
if (block != null)
{
if (DumpNULL || !UnsafeHelp.IsZero(block))
{
Buffer.BlockCopy(block, 0, bpage, 0, 4096);
//Buffer.MemoryCopy(lp, bp, 4096, 4096);
using (var savefile = (canAppend ? File.Open(lastLoc, FileMode.Append, FileAccess.Write, FileShare.ReadWrite) : File.OpenWrite(saveLoc)))
savefile.Write(bpage, 0, 4096);
//savefile.Write(bpage, 0, 4096);
return lastLoc;
}
}
}
ContigSize = 0;
return string.Empty;
//}
//}
}
private void Initialize(string filename, ushort blocksize)
{
if (File.Exists(filename) == false)
_datawrite = new FileStream(filename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite);
else
_datawrite = new FileStream(filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
if (_datawrite.Length == 0)
{
CreateFileHeader(blocksize);
// new file
_datawrite.Write(_fileheader, 0, _fileheader.Length);
_datawrite.Flush();
}
else
{
ReadFileHeader();
_lastBlockNumber = (int)((_datawrite.Length - _fileheader.Length) / _BLOCKSIZE);
_lastBlockNumber++;
}
_freeList = new WAHBitArray();
if (File.Exists(_Path + _filename + ".free"))
{
ReadFreeListBMPFile(_Path + _filename + ".free");
// delete file so if failure no big deal on restart
File.Delete(_Path + _filename + ".free");
}
}
public WAHBitArray Xor(WAHBitArray op)
{
lock (_lock)
{
uint[] left;
uint[] right;
prelogic(op, out left, out right);
for (int i = 0; i < left.Length; i++)
left[i] ^= right[i];
return new WAHBitArray(TYPE.Bitarray, left);
}
}
public WAHBitArray Query(RDBExpression ex, object from, int maxsize)
{
// always return everything
return(WAHBitArray.Fill(maxsize));
}
public WAHBitArray Query(object fromkey, object tokey, int maxsize)
{
return(WAHBitArray.Fill(maxsize)); // TODO : all or none??
}
private void prelogic(WAHBitArray op, out uint[] left, out uint[] right)
{
this.CheckBitArray();
left = this.GetBitArray();
right = op.GetBitArray();
int ic = left.Length;
int uc = right.Length;
if (ic > uc)
{
uint[] ar = new uint[ic];
right.CopyTo(ar, 0);
right = ar;
}
else if (ic < uc)
{
uint[] ar = new uint[uc];
left.CopyTo(ar, 0);
left = ar;
}
}
public void InPlaceOR(WAHBitArray left)
{
_bits = _bits.Or(left);
}
public static Mem InitMem(String mFile, AMemoryRunDetector Detector, uint[] BitmapArray = null) //: this()
{
var thiz = new Mem();
thiz.StartOfMemory = Detector != null ? Detector.StartOfMem : 0;
if (Detector != null)
{
thiz.StartOfMemory = Detector.StartOfMem;
thiz.MD = Detector;
}
#if USE_BITMAP
// maybe there's a bit map we can use from a DMP file
if (BitmapArray != null)
pfnTableIdx = new WAHBitArray(WAHBitArray.TYPE.Bitarray, BitmapArray);
else
pfnTableIdx = new WAHBitArray();
// 32bit's of pages should be plenty?
pfnTableIdx.Length = (int) (MapViewSize / 0x1000);
#endif
if (File.Exists(mFile))
{
thiz.MemoryDump = mFile;
thiz.FileSize = new FileInfo(mFile).Length;
if (Detector != null)
thiz.MD = Detector;
}
thiz.SetupStreams();
return thiz;
}
