public void TestWAHBitArray4()
{
var b = new WAHBitArray();
int count = 25;
for (int i = 0; i < 5; i++)
{
b.Set(i, true);
}
for (int i = 5; i < count + 5; i++)
{
b.Set(i, false);
}
for (int i = 30; i < 64; i++)
{
b.Set(i, i != 35);
}
for (int i = 64; i < 100; i++)
{
b.Set(i, true);
}
var expected = b.GetBitIndexes().ToArray();
var b2 = new WAHBitArray(WAHBitArray.TYPE.Compressed_WAH, b.GetCompressed());
var actual = b2.GetBitIndexes().ToArray();
expected.Should(Be.EqualTo(actual));
}
internal int ViewDelete(string filter)
{
_log.Debug("delete : " + _view.Name);
if (_isDirty == false)
{
WriteDirtyFile();
}
int count = 0;
if (filter != "")
{
LambdaExpression le = null;
if (_lambdacache.TryGetValue(filter, out le) == false)
{
le = System.Linq.Dynamic.DynamicExpression.ParseLambda(_view.Schema, typeof(bool), filter, null);
_lambdacache.Add(filter, le);
}
QueryVisitor qv = new QueryVisitor(QueryColumnExpression);
qv.Visit(le.Body);
count = qv._bitmap.Count;
if (count > 0)
{
WAHBitArray qbits = (WAHBitArray)qv._bitmap.Pop();
_deletedRows.InPlaceOR(qbits);
count = (int)qbits.CountOnes();
}
}
return(count);
}
internal int Count(string filter)
{
int totcount = 0;
DateTime dt = FastDateTime.Now;
filter = filter.Trim();
if (filter == null || filter == "")
{
totcount = internalCount();
}
else
{
_log.Debug("Count filter : " + filter);
WAHBitArray ba = new WAHBitArray();
LambdaExpression le = null;
if (_lambdacache.TryGetValue(filter, out le) == false)
{
le = System.Linq.Dynamic.DynamicExpression.ParseLambda(_view.Schema, typeof(bool), filter, null);
_lambdacache.Add(filter, le);
}
QueryVisitor qv = new QueryVisitor(QueryColumnExpression);
qv.Visit(le.Body);
var delbits = _deletedRows.GetBits();
ba = ((WAHBitArray)qv._bitmap.Pop()).AndNot(delbits);
totcount = (int)ba.CountOnes();
}
_log.Debug("Count items = " + totcount);
_log.Debug("Count time (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
return(totcount);
}
private void DeleteRowsWith(Guid guid)
{
// find bitmap for guid column
WAHBitArray gc = QueryColumnExpression(_docid, RDBExpression.Equal, guid);
_deletedRows.InPlaceOR(gc);
}
public override Expression Visit(Expression node)
{
if (node.NodeType == ExpressionType.NewArrayInit)
{
var a = node as NewArrayExpression;
_count = a.Expressions.Count;
return(base.Visit(node));
}
else if (node.NodeType == ExpressionType.MemberAccess)
{
var v = base.Visit(node);
if (_InCommand != "")
{
var a = _stack.Pop() as IList;
foreach (var c in a)
{
if (_inBmp == null)
{
_inBmp = qexpression(_InCommand, RDBExpression.Equal, c);
}
else
{
_inBmp = _inBmp.Or(qexpression(_InCommand, RDBExpression.Equal, c));
}
}
}
return(v);
}
else
{
return(base.Visit(node));
}
}
public static void WAH_test()
{
int count = 500;
for (int i = 0; i < 1000; i++)
{
Console.Write(".");
if (i % 80 == 0)
{
Console.WriteLine();
}
WAHBitArray ba = new WAHBitArray();
for (int j = 0; j < count; j++)
{
ba.Set(i * count + j, true);
}
WAHBitArray.TYPE type;
var bits = ba.GetCompressed(out type);
var ba2 = new WAHBitArray(type, bits);
var res = ba2.Xor(ba);
long c = res.CountOnes();
Assert.AreEqual(0, c);
}
//Random r = new Random();
//WAHBitArray b = new WAHBitArray();
//for (int i = 0; i < 100000; i++)
//{
// b.Set(
//}
}
public VirtualScanner()
{
DetectedFragments = new ConcurrentDictionary <long, VAScanType>();
Artifacts = new ConcurrentDictionary <long, Extract>();
CheckMethods = new List <Func <long, long[], VAScanType> >();
ScanList = new WAHBitArray();
}
public void TestWAHBitArray1()
{
var b = new WAHBitArray();
int count = 31;
for (int i = 0; i < count; i++)
{
b.Set(i, true);
}
var expected = b.GetBitIndexes().ToArray();
var b2 = new WAHBitArray(WAHBitArray.TYPE.Compressed_WAH, b.GetCompressed());
var actual = b2.GetBitIndexes().ToArray();
expected.Should(Be.EqualTo(actual));
}
public QueryPlanResult Eval()
{
List <LockToken> locks = _locks.OrderBy(i => i.Token).Select(o => o.Lock()).ToList();
try
{
WAHBitArray result = _evalCore();
return(new QueryPlanResult(result, locks.ToDictionary(l => l.TypeToken, l => l.Version)));
}
finally
{
locks.Reverse();
locks.ForEach(l => l.Token.Dispose());
}
}
// FEATURE : add query caching here
internal Result <object> Query <T>(Expression <Predicate <T> > filter, int start, int count, string orderby)
{
if (filter == null)
{
return(Query(start, count));
}
DateTime dt = FastDateTime.Now;
_log.Debug("query : " + _view.Name);
WAHBitArray ba = new WAHBitArray();
QueryVisitor qv = new QueryVisitor(QueryColumnExpression);
qv.Visit(filter);
var delbits = _deletedRows.GetBits();
ba = ((WAHBitArray)qv._bitmap.Pop()).AndNot(delbits);
List <T> trows = null;
if (_viewmanager.inTransaction())
{
// query from transaction own data
tran_data data = null;
if (_transactions.TryGetValue(Thread.CurrentThread.ManagedThreadId, out data))
{
List <T> rrows = new List <T>();
foreach (var kv in data.rows)
{
foreach (var r in kv.Value)
{
object o = FastCreateObject(_view.Schema);
rrows.Add((T)_rowfiller(o, r));
}
}
trows = rrows.FindAll(filter.Compile());
}
}
var order = SortBy(orderby);
_log.Debug("query bitmap done (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
dt = FastDateTime.Now;
// exec query return rows
return(ReturnRows <T>(ba, trows, start, count, order));
}
internal Result <object> Query(string filter, int start, int count, string orderby)
{
filter = filter.Trim();
if (filter == "")
{
return(Query(start, count, orderby));
}
DateTime dt = FastDateTime.Now;
_log.Debug("query : " + _view.Name);
_log.Debug("query : " + filter);
_log.Debug("orderby : " + orderby);
WAHBitArray ba = new WAHBitArray();
var delbits = _deletedRows.GetBits();
if (filter != "")
{
LambdaExpression le = null;
if (_lambdacache.TryGetValue(filter, out le) == false)
{
le = System.Linq.Dynamic.DynamicExpression.ParseLambda(_view.Schema, typeof(bool), filter, null);
_lambdacache.Add(filter, le);
}
QueryVisitor qv = new QueryVisitor(QueryColumnExpression);
qv.Visit(le.Body);
ba = ((WAHBitArray)qv._bitmap.Pop()).AndNot(delbits);
}
else
{
ba = WAHBitArray.Fill(_viewData.Count()).AndNot(delbits);
}
var order = SortBy(orderby);
_log.Debug("query bitmap done (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
dt = FastDateTime.Now;
// exec query return rows
return(ReturnRows <object>(ba, null, start, count, order));
}
public static Mem InitMem(String mFile, AMemoryRunDetector Detector, uint[] BitmapArray = null) //: this()
{
var thiz = new Mem();
thiz.StartOfMemory = Detector != null ? Detector.StartOfMem : 0;
if (Detector != null)
{
thiz.StartOfMemory = Detector.StartOfMem;
thiz.MD = Detector;
}
#if USE_BITMAP
// maybe there's a bit map we can use from a DMP file
if (BitmapArray != null)
{
pfnTableIdx = new WAHBitArray(WAHBitArray.TYPE.Bitarray, BitmapArray);
}
else
{
pfnTableIdx = new WAHBitArray();
}
// 32bit's of pages should be plenty?
pfnTableIdx.Length = (int)(MapViewSize / 0x1000);
#endif
if (File.Exists(mFile))
{
thiz.MemoryDump = mFile;
thiz.FileSize = new FileInfo(mFile).Length;
if (Detector != null)
{
thiz.MD = Detector;
}
}
thiz.SetupStreams();
return(thiz);
}
internal int ViewDelete <T>(Expression <Predicate <T> > filter)
{
_log.Debug("delete : " + _view.Name);
if (_isDirty == false)
{
WriteDirtyFile();
}
QueryVisitor qv = new QueryVisitor(QueryColumnExpression);
qv.Visit(filter);
var delbits = _deletedRows.GetBits();
int count = qv._bitmap.Count;
if (count > 0)
{
WAHBitArray qbits = (WAHBitArray)qv._bitmap.Pop();
_deletedRows.InPlaceOR(qbits);
count = (int)qbits.CountOnes();
}
_log.Debug("Deleted rows = " + count);
return(count);
}
protected override Expression VisitConstant(ConstantExpression c)
{
IQueryable q = c.Value as IQueryable;
if (q != null)
{
_stack.Push(q.ElementType.Name);
}
else if (c.Value == null)
{
_stack.Push(null);
}
else
{
Type t = c.Value.GetType();
if (t.IsValueType || t == typeof(string))
{
if (_InCommand != "")
{
if (_inBmp == null)
{
_inBmp = qexpression(_InCommand, RDBExpression.Equal, c.Value);
}
else
{
_inBmp = _inBmp.Or(qexpression(_InCommand, RDBExpression.Equal, c.Value));
}
}
else
{
_stack.Push(c.Value);
}
}
}
return(c);
}
internal int Count <T>(Expression <Predicate <T> > filter)
{
int totcount = 0;
DateTime dt = FastDateTime.Now;
if (filter == null)
{
totcount = internalCount();
}
else
{
WAHBitArray ba = new WAHBitArray();
QueryVisitor qv = new QueryVisitor(QueryColumnExpression);
qv.Visit(filter);
var delbits = _deletedRows.GetBits();
ba = ((WAHBitArray)qv._bitmap.Pop()).AndNot(delbits);
totcount = (int)ba.CountOnes();
}
_log.Debug("Count items = " + totcount);
_log.Debug("Count time (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
return(totcount);
}
internal Result <object> Query(int start, int count, string orderby)
{
// no filter query -> just show all the data
DateTime dt = FastDateTime.Now;
_log.Debug("query : " + _view.Name);
int totalviewrows = _viewData.Count();
List <object> rows = new List <object>();
Result <object> ret = new Result <object>();
int skip = start;
int cc = 0;
WAHBitArray del = _deletedRows.GetBits();
ret.TotalCount = totalviewrows - (int)del.CountOnes();
var order = SortBy(orderby);
if (order.Count == 0)
{
for (int i = 0; i < totalviewrows; i++)
{
order.Add(i);
}
}
if (count == -1)
{
count = totalviewrows;
}
foreach (int i in order)
{
if (del.Get(i) == true)
{
continue;
}
if (skip > 0)
{
skip--;
}
else
{
bool b = OutputRow <object>(rows, i);
if (b && count > 0)
{
cc++;
}
}
if (cc == count)
{
break;
}
}
_log.Debug("query rows fetched (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
_log.Debug("query rows count : " + rows.Count.ToString("#,0"));
ret.OK = true;
ret.Count = rows.Count;
//ret.TotalCount = rows.Count;
ret.Rows = rows;
return(ret);
}
protected override Expression VisitBinary(BinaryExpression b)
{
_leftmode = true;
var m = this.Visit(b.Left);
if (m == null) // VB.net sty;e linq for string compare
{
return(b.Right);
}
ExpressionType t = b.NodeType;
if (t == ExpressionType.Equal || t == ExpressionType.NotEqual ||
t == ExpressionType.LessThan || t == ExpressionType.LessThanOrEqual ||
t == ExpressionType.GreaterThan || t == ExpressionType.GreaterThanOrEqual)
{
_stack.Push(b.NodeType);
}
_leftmode = false;
this.Visit(b.Right);
t = b.NodeType;
if (t == ExpressionType.Equal || t == ExpressionType.NotEqual ||
t == ExpressionType.LessThanOrEqual || t == ExpressionType.LessThan ||
t == ExpressionType.GreaterThanOrEqual || t == ExpressionType.GreaterThan
)
{
// binary expression
object lval = _stack.Pop();
ExpressionType lop = (ExpressionType)_stack.Pop();
string lname = (string)_stack.Pop();
if (_stack.Count > 0)
{
lname += "_" + (string)_stack.Pop();
}
RDBExpression exp = RDBExpression.Equal;
if (lop == ExpressionType.LessThan)
{
exp = RDBExpression.Less;
}
else if (lop == ExpressionType.LessThanOrEqual)
{
exp = RDBExpression.LessEqual;
}
else if (lop == ExpressionType.GreaterThan)
{
exp = RDBExpression.Greater;
}
else if (lop == ExpressionType.GreaterThanOrEqual)
{
exp = RDBExpression.GreaterEqual;
}
else if (lop == ExpressionType.NotEqual)
{
exp = RDBExpression.NotEqual;
}
_bitmap.Push(qexpression(lname, exp, lval));
}
if (t == ExpressionType.And || t == ExpressionType.AndAlso ||
t == ExpressionType.Or || t == ExpressionType.OrElse)
{
if (_bitmap.Count > 1)
{
// do bitmap operations
WAHBitArray right = (WAHBitArray)_bitmap.Pop();
WAHBitArray left = (WAHBitArray)_bitmap.Pop();
if (t == ExpressionType.And || t == ExpressionType.AndAlso)
{
_bitmap.Push(right.And(left));
}
if (t == ExpressionType.Or || t == ExpressionType.OrElse)
{
_bitmap.Push(right.Or(left));
}
}
else
{
// single bitmap operation
}
}
return(b);
}
private static WAHBitArray DoBitOperation(WAHBitArray bits, WAHBitArray c, OPERATION op, int maxsize)
{
if (bits != null)
{
switch (op)
{
case OPERATION.AND:
bits = c.And(bits);
break;
case OPERATION.OR:
bits = c.Or(bits);
break;
case OPERATION.ANDNOT:
bits = c.And(bits.Not(maxsize));
break;
}
}
else
bits = c;
return bits;
}
protected override Expression VisitBinary(BinaryExpression b)
{
var m = this.Visit(b.Left);
if (m == null) // VB.net sty;e linq for string compare
{
return(b.Right);
}
ExpressionType t = b.NodeType;
if (t == ExpressionType.Equal || t == ExpressionType.NotEqual ||
t == ExpressionType.LessThan || t == ExpressionType.LessThanOrEqual ||
t == ExpressionType.GreaterThan || t == ExpressionType.GreaterThanOrEqual)
{
_stack.Push(b.NodeType);
}
this.Visit(b.Right);
t = b.NodeType;
if (t == ExpressionType.Equal || t == ExpressionType.NotEqual ||
t == ExpressionType.LessThanOrEqual || t == ExpressionType.LessThan ||
t == ExpressionType.GreaterThanOrEqual || t == ExpressionType.GreaterThan
)
{
// binary expression
object lv = _stack.Pop();
ExpressionType lo = (ExpressionType)_stack.Pop();
object ln = _stack.Pop();
RDBExpression exp = RDBExpression.Equal;
// FEATURE : add contains , between, startswith
if (lo == ExpressionType.LessThan)
{
exp = RDBExpression.Less;
}
else if (lo == ExpressionType.LessThanOrEqual)
{
exp = RDBExpression.LessEqual;
}
else if (lo == ExpressionType.GreaterThan)
{
exp = RDBExpression.Greater;
}
else if (lo == ExpressionType.GreaterThanOrEqual)
{
exp = RDBExpression.GreaterEqual;
}
else if (lo == ExpressionType.NotEqual)
{
exp = RDBExpression.NotEqual;
}
_bitmap.Push(qexpression("" + ln, exp, lv));
}
if (t == ExpressionType.And || t == ExpressionType.AndAlso ||
t == ExpressionType.Or || t == ExpressionType.OrElse)
{
if (_bitmap.Count > 1)
{
// do bitmap operations
WAHBitArray r = (WAHBitArray)_bitmap.Pop();
WAHBitArray l = (WAHBitArray)_bitmap.Pop();
if (t == ExpressionType.And || t == ExpressionType.AndAlso)
{
_bitmap.Push(r.And(l));
}
if (t == ExpressionType.Or || t == ExpressionType.OrElse)
{
_bitmap.Push(r.Or(l));
}
}
else
{
// single bitmap operation
}
}
return(b);
}
public Mem(String mFile, uint[] BitmapArray = null, MemoryDescriptor Override = null)
{
GapScanSize = 0x10000000;
StartOfMemory = Override != null ? Override.StartOfMemmory : 0;
MapViewBase = 0;
MapViewSize = MapWindowSize = (0x1000 * 0x1000 * 4); // Due to the physical page allocation algorithm a modest window is probably fine
// so not even 1/2 the size of the window which was only getting < 50% hit ratio at best
// PageCache may be better off than a huge window...
if (PageCache == null)
{
PageCache = new ConcurrentDictionary <long, long[]>(8, PageCacheMax);
}
if (BitmapArray != null)
{
pfnTableIdx = new WAHBitArray(WAHBitArray.TYPE.Bitarray, BitmapArray);
}
else
{
pfnTableIdx = new WAHBitArray();
}
// 32bit's of pages should be plenty?
pfnTableIdx.Length = (int)(MapViewSize / 0x1000);
if (File.Exists(mFile))
{
MemoryDump = mFile;
FileSize = new FileInfo(MemoryDump).Length;
if (Override != null)
{
MD = Override;
}
else
{
MD = new MemoryDescriptor(FileSize);
if (DetectedDescriptor != null)
{
MD = DetectedDescriptor;
}
}
MapViewSize = FileSize < MapWindowSize ? FileSize : MapWindowSize;
var lmindex = Interlocked.Increment(ref mindex);
var mapName = Path.GetFileNameWithoutExtension(MemoryDump) + lmindex.ToString();
mapStream = new FileStream(MemoryDump, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
mappedFile = MemoryMappedFile.CreateFromFile(mapStream,
mapName,
0,
MemoryMappedFileAccess.Read,
null,
HandleInheritability.Inheritable,
false);
mappedAccess = mappedFile.CreateViewAccessor(
MapViewBase,
MapViewSize,
MemoryMappedFileAccess.Read);
DiscoveredGaps = new Dictionary <long, long>();
}
}
public QueryPlanResult(WAHBitArray bitArray, IEnumerable <KeyValuePair <DocumentIndexTypeToken, int> > versions)
{
_bitArray = bitArray;
_versions = versions.ToDictionary(p => p.Key, p => p.Value);
}
private Result <T> ReturnRows2 <T>(WAHBitArray ba, List <T> trows, int start, int count, List <int> orderby)
{
DateTime dt = FastDateTime.Now;
List <T> rows = new List <T>();
Result <T> ret = new Result <T>();
int skip = start;
int c = 0;
ret.TotalCount = (int)ba.CountOnes();
if (count == -1)
{
count = ret.TotalCount;
}
if (count > 0)
{
foreach (int i in orderby)
{
if (ba.Get(i))
{
if (skip > 0)
{
skip--;
}
else
{
bool b = OutputRow <T>(rows, i);
if (b && count > 0)
{
c++;
}
}
ba.Set(i, false);
if (c == count)
{
break;
}
}
}
foreach (int i in ba.GetBitIndexes())
{
if (c < count)
{
if (skip > 0)
{
skip--;
}
else
{
bool b = OutputRow <T>(rows, i);
if (b && count > 0)
{
c++;
}
}
if (c == count)
{
break;
}
}
}
}
if (trows != null)// TODO : move to start and decrement in count
{
foreach (var o in trows)
{
rows.Add(o);
}
}
_log.Debug("query rows fetched (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
_log.Debug("query rows count : " + rows.Count.ToString("#,0"));
ret.OK = true;
ret.Count = rows.Count;
ret.Rows = rows;
return(ret);
}