public void TestWAHBitArray4()
        {
            var b     = new WAHBitArray();
            int count = 25;

            for (int i = 0; i < 5; i++)
            {
                b.Set(i, true);
            }
            for (int i = 5; i < count + 5; i++)
            {
                b.Set(i, false);
            }
            for (int i = 30; i < 64; i++)
            {
                b.Set(i, i != 35);
            }
            for (int i = 64; i < 100; i++)
            {
                b.Set(i, true);
            }
            var expected = b.GetBitIndexes().ToArray();
            var b2       = new WAHBitArray(WAHBitArray.TYPE.Compressed_WAH, b.GetCompressed());
            var actual   = b2.GetBitIndexes().ToArray();

            expected.Should(Be.EqualTo(actual));
        }
Пример #2
0
        internal int ViewDelete(string filter)
        {
            _log.Debug("delete : " + _view.Name);
            if (_isDirty == false)
            {
                WriteDirtyFile();
            }
            int count = 0;

            if (filter != "")
            {
                LambdaExpression le = null;
                if (_lambdacache.TryGetValue(filter, out le) == false)
                {
                    le = System.Linq.Dynamic.DynamicExpression.ParseLambda(_view.Schema, typeof(bool), filter, null);
                    _lambdacache.Add(filter, le);
                }
                QueryVisitor qv = new QueryVisitor(QueryColumnExpression);
                qv.Visit(le.Body);
                count = qv._bitmap.Count;
                if (count > 0)
                {
                    WAHBitArray qbits = (WAHBitArray)qv._bitmap.Pop();
                    _deletedRows.InPlaceOR(qbits);
                    count = (int)qbits.CountOnes();
                }
            }
            return(count);
        }
Пример #3
0
        internal int Count(string filter)
        {
            int      totcount = 0;
            DateTime dt       = FastDateTime.Now;

            filter = filter.Trim();
            if (filter == null || filter == "")
            {
                totcount = internalCount();
            }
            else
            {
                _log.Debug("Count filter : " + filter);
                WAHBitArray ba = new WAHBitArray();

                LambdaExpression le = null;
                if (_lambdacache.TryGetValue(filter, out le) == false)
                {
                    le = System.Linq.Dynamic.DynamicExpression.ParseLambda(_view.Schema, typeof(bool), filter, null);
                    _lambdacache.Add(filter, le);
                }
                QueryVisitor qv = new QueryVisitor(QueryColumnExpression);
                qv.Visit(le.Body);
                var delbits = _deletedRows.GetBits();
                ba = ((WAHBitArray)qv._bitmap.Pop()).AndNot(delbits);

                totcount = (int)ba.CountOnes();
            }
            _log.Debug("Count items = " + totcount);
            _log.Debug("Count time (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
            return(totcount);
        }
Пример #4
0
        private void DeleteRowsWith(Guid guid)
        {
            // find bitmap for guid column
            WAHBitArray gc = QueryColumnExpression(_docid, RDBExpression.Equal, guid);

            _deletedRows.InPlaceOR(gc);
        }
Пример #5
0
        public override Expression Visit(Expression node)
        {
            if (node.NodeType == ExpressionType.NewArrayInit)
            {
                var a = node as NewArrayExpression;
                _count = a.Expressions.Count;
                return(base.Visit(node));
            }
            else if (node.NodeType == ExpressionType.MemberAccess)
            {
                var v = base.Visit(node);
                if (_InCommand != "")
                {
                    var a = _stack.Pop() as IList;
                    foreach (var c in a)
                    {
                        if (_inBmp == null)
                        {
                            _inBmp = qexpression(_InCommand, RDBExpression.Equal, c);
                        }
                        else
                        {
                            _inBmp = _inBmp.Or(qexpression(_InCommand, RDBExpression.Equal, c));
                        }
                    }
                }

                return(v);
            }
            else
            {
                return(base.Visit(node));
            }
        }
Пример #6
0
        public static void WAH_test()
        {
            int count = 500;

            for (int i = 0; i < 1000; i++)
            {
                Console.Write(".");
                if (i % 80 == 0)
                {
                    Console.WriteLine();
                }
                WAHBitArray ba = new WAHBitArray();
                for (int j = 0; j < count; j++)
                {
                    ba.Set(i * count + j, true);
                }

                WAHBitArray.TYPE type;
                var bits = ba.GetCompressed(out type);

                var ba2 = new WAHBitArray(type, bits);

                var  res = ba2.Xor(ba);
                long c   = res.CountOnes();
                Assert.AreEqual(0, c);
            }

            //Random r = new Random();
            //WAHBitArray b = new WAHBitArray();
            //for (int i = 0; i < 100000; i++)
            //{
            //    b.Set(
            //}
        }
Пример #7
0
        public VirtualScanner()
        {
            DetectedFragments = new ConcurrentDictionary <long, VAScanType>();
            Artifacts         = new ConcurrentDictionary <long, Extract>();

            CheckMethods = new List <Func <long, long[], VAScanType> >();
            ScanList     = new WAHBitArray();
        }
        public void TestWAHBitArray1()
        {
            var b     = new WAHBitArray();
            int count = 31;

            for (int i = 0; i < count; i++)
            {
                b.Set(i, true);
            }
            var expected = b.GetBitIndexes().ToArray();
            var b2       = new WAHBitArray(WAHBitArray.TYPE.Compressed_WAH, b.GetCompressed());
            var actual   = b2.GetBitIndexes().ToArray();

            expected.Should(Be.EqualTo(actual));
        }
Пример #9
0
        public QueryPlanResult Eval()
        {
            List <LockToken> locks = _locks.OrderBy(i => i.Token).Select(o => o.Lock()).ToList();

            try
            {
                WAHBitArray result = _evalCore();
                return(new QueryPlanResult(result, locks.ToDictionary(l => l.TypeToken, l => l.Version)));
            }
            finally
            {
                locks.Reverse();
                locks.ForEach(l => l.Token.Dispose());
            }
        }
Пример #10
0
        // FEATURE : add query caching here
        internal Result <object> Query <T>(Expression <Predicate <T> > filter, int start, int count, string orderby)
        {
            if (filter == null)
            {
                return(Query(start, count));
            }

            DateTime dt = FastDateTime.Now;

            _log.Debug("query : " + _view.Name);

            WAHBitArray ba = new WAHBitArray();

            QueryVisitor qv = new QueryVisitor(QueryColumnExpression);

            qv.Visit(filter);
            var delbits = _deletedRows.GetBits();

            ba = ((WAHBitArray)qv._bitmap.Pop()).AndNot(delbits);
            List <T> trows = null;

            if (_viewmanager.inTransaction())
            {
                // query from transaction own data
                tran_data data = null;
                if (_transactions.TryGetValue(Thread.CurrentThread.ManagedThreadId, out data))
                {
                    List <T> rrows = new List <T>();
                    foreach (var kv in data.rows)
                    {
                        foreach (var r in kv.Value)
                        {
                            object o = FastCreateObject(_view.Schema);
                            rrows.Add((T)_rowfiller(o, r));
                        }
                    }
                    trows = rrows.FindAll(filter.Compile());
                }
            }

            var order = SortBy(orderby);

            _log.Debug("query bitmap done (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
            dt = FastDateTime.Now;
            // exec query return rows
            return(ReturnRows <T>(ba, trows, start, count, order));
        }
Пример #11
0
        internal Result <object> Query(string filter, int start, int count, string orderby)
        {
            filter = filter.Trim();
            if (filter == "")
            {
                return(Query(start, count, orderby));
            }

            DateTime dt = FastDateTime.Now;

            _log.Debug("query : " + _view.Name);
            _log.Debug("query : " + filter);
            _log.Debug("orderby : " + orderby);

            WAHBitArray ba      = new WAHBitArray();
            var         delbits = _deletedRows.GetBits();

            if (filter != "")
            {
                LambdaExpression le = null;
                if (_lambdacache.TryGetValue(filter, out le) == false)
                {
                    le = System.Linq.Dynamic.DynamicExpression.ParseLambda(_view.Schema, typeof(bool), filter, null);
                    _lambdacache.Add(filter, le);
                }
                QueryVisitor qv = new QueryVisitor(QueryColumnExpression);
                qv.Visit(le.Body);

                ba = ((WAHBitArray)qv._bitmap.Pop()).AndNot(delbits);
            }
            else
            {
                ba = WAHBitArray.Fill(_viewData.Count()).AndNot(delbits);
            }

            var order = SortBy(orderby);

            _log.Debug("query bitmap done (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
            dt = FastDateTime.Now;
            // exec query return rows
            return(ReturnRows <object>(ba, null, start, count, order));
        }
Пример #12
0
        public static Mem InitMem(String mFile, AMemoryRunDetector Detector, uint[] BitmapArray = null) //: this()
        {
            var thiz = new Mem();

            thiz.StartOfMemory = Detector != null ? Detector.StartOfMem : 0;

            if (Detector != null)
            {
                thiz.StartOfMemory = Detector.StartOfMem;
                thiz.MD            = Detector;
            }
#if USE_BITMAP
            // maybe there's a bit map we can use from a DMP file
            if (BitmapArray != null)
            {
                pfnTableIdx = new WAHBitArray(WAHBitArray.TYPE.Bitarray, BitmapArray);
            }
            else
            {
                pfnTableIdx = new WAHBitArray();
            }

            // 32bit's of pages should be plenty?
            pfnTableIdx.Length = (int)(MapViewSize / 0x1000);
#endif

            if (File.Exists(mFile))
            {
                thiz.MemoryDump = mFile;
                thiz.FileSize   = new FileInfo(mFile).Length;

                if (Detector != null)
                {
                    thiz.MD = Detector;
                }
            }

            thiz.SetupStreams();

            return(thiz);
        }
Пример #13
0
        internal int ViewDelete <T>(Expression <Predicate <T> > filter)
        {
            _log.Debug("delete : " + _view.Name);
            if (_isDirty == false)
            {
                WriteDirtyFile();
            }
            QueryVisitor qv = new QueryVisitor(QueryColumnExpression);

            qv.Visit(filter);
            var delbits = _deletedRows.GetBits();
            int count   = qv._bitmap.Count;

            if (count > 0)
            {
                WAHBitArray qbits = (WAHBitArray)qv._bitmap.Pop();
                _deletedRows.InPlaceOR(qbits);
                count = (int)qbits.CountOnes();
            }
            _log.Debug("Deleted rows = " + count);
            return(count);
        }
Пример #14
0
        protected override Expression VisitConstant(ConstantExpression c)
        {
            IQueryable q = c.Value as IQueryable;

            if (q != null)
            {
                _stack.Push(q.ElementType.Name);
            }
            else if (c.Value == null)
            {
                _stack.Push(null);
            }
            else
            {
                Type t = c.Value.GetType();
                if (t.IsValueType || t == typeof(string))
                {
                    if (_InCommand != "")
                    {
                        if (_inBmp == null)
                        {
                            _inBmp = qexpression(_InCommand, RDBExpression.Equal, c.Value);
                        }
                        else
                        {
                            _inBmp = _inBmp.Or(qexpression(_InCommand, RDBExpression.Equal, c.Value));
                        }
                    }
                    else
                    {
                        _stack.Push(c.Value);
                    }
                }
            }
            return(c);
        }
Пример #15
0
        internal int Count <T>(Expression <Predicate <T> > filter)
        {
            int      totcount = 0;
            DateTime dt       = FastDateTime.Now;

            if (filter == null)
            {
                totcount = internalCount();
            }
            else
            {
                WAHBitArray ba = new WAHBitArray();

                QueryVisitor qv = new QueryVisitor(QueryColumnExpression);
                qv.Visit(filter);
                var delbits = _deletedRows.GetBits();
                ba = ((WAHBitArray)qv._bitmap.Pop()).AndNot(delbits);

                totcount = (int)ba.CountOnes();
            }
            _log.Debug("Count items = " + totcount);
            _log.Debug("Count time (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
            return(totcount);
        }
Пример #16
0
        internal Result <object> Query(int start, int count, string orderby)
        {
            // no filter query -> just show all the data
            DateTime dt = FastDateTime.Now;

            _log.Debug("query : " + _view.Name);
            int             totalviewrows = _viewData.Count();
            List <object>   rows          = new List <object>();
            Result <object> ret           = new Result <object>();
            int             skip          = start;
            int             cc            = 0;
            WAHBitArray     del           = _deletedRows.GetBits();

            ret.TotalCount = totalviewrows - (int)del.CountOnes();

            var order = SortBy(orderby);

            if (order.Count == 0)
            {
                for (int i = 0; i < totalviewrows; i++)
                {
                    order.Add(i);
                }
            }

            if (count == -1)
            {
                count = totalviewrows;
            }

            foreach (int i in order)
            {
                if (del.Get(i) == true)
                {
                    continue;
                }
                if (skip > 0)
                {
                    skip--;
                }
                else
                {
                    bool b = OutputRow <object>(rows, i);
                    if (b && count > 0)
                    {
                        cc++;
                    }
                }
                if (cc == count)
                {
                    break;
                }
            }

            _log.Debug("query rows fetched (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
            _log.Debug("query rows count : " + rows.Count.ToString("#,0"));
            ret.OK    = true;
            ret.Count = rows.Count;
            //ret.TotalCount = rows.Count;
            ret.Rows = rows;
            return(ret);
        }
Пример #17
0
        protected override Expression VisitBinary(BinaryExpression b)
        {
            _leftmode = true;
            var m = this.Visit(b.Left);

            if (m == null) // VB.net sty;e linq for string compare
            {
                return(b.Right);
            }
            ExpressionType t = b.NodeType;

            if (t == ExpressionType.Equal || t == ExpressionType.NotEqual ||
                t == ExpressionType.LessThan || t == ExpressionType.LessThanOrEqual ||
                t == ExpressionType.GreaterThan || t == ExpressionType.GreaterThanOrEqual)
            {
                _stack.Push(b.NodeType);
            }

            _leftmode = false;
            this.Visit(b.Right);
            t = b.NodeType;
            if (t == ExpressionType.Equal || t == ExpressionType.NotEqual ||
                t == ExpressionType.LessThanOrEqual || t == ExpressionType.LessThan ||
                t == ExpressionType.GreaterThanOrEqual || t == ExpressionType.GreaterThan
                )
            {
                // binary expression
                object         lval  = _stack.Pop();
                ExpressionType lop   = (ExpressionType)_stack.Pop();
                string         lname = (string)_stack.Pop();
                if (_stack.Count > 0)
                {
                    lname += "_" + (string)_stack.Pop();
                }
                RDBExpression exp = RDBExpression.Equal;
                if (lop == ExpressionType.LessThan)
                {
                    exp = RDBExpression.Less;
                }
                else if (lop == ExpressionType.LessThanOrEqual)
                {
                    exp = RDBExpression.LessEqual;
                }
                else if (lop == ExpressionType.GreaterThan)
                {
                    exp = RDBExpression.Greater;
                }
                else if (lop == ExpressionType.GreaterThanOrEqual)
                {
                    exp = RDBExpression.GreaterEqual;
                }
                else if (lop == ExpressionType.NotEqual)
                {
                    exp = RDBExpression.NotEqual;
                }

                _bitmap.Push(qexpression(lname, exp, lval));
            }

            if (t == ExpressionType.And || t == ExpressionType.AndAlso ||
                t == ExpressionType.Or || t == ExpressionType.OrElse)
            {
                if (_bitmap.Count > 1)
                {
                    // do bitmap operations
                    WAHBitArray right = (WAHBitArray)_bitmap.Pop();
                    WAHBitArray left  = (WAHBitArray)_bitmap.Pop();

                    if (t == ExpressionType.And || t == ExpressionType.AndAlso)
                    {
                        _bitmap.Push(right.And(left));
                    }
                    if (t == ExpressionType.Or || t == ExpressionType.OrElse)
                    {
                        _bitmap.Push(right.Or(left));
                    }
                }
                else
                {
                    // single bitmap operation
                }
            }
            return(b);
        }
Пример #18
0
 private static WAHBitArray DoBitOperation(WAHBitArray bits, WAHBitArray c, OPERATION op, int maxsize)
 {
     if (bits != null)
     {
         switch (op)
         {
             case OPERATION.AND:
                 bits = c.And(bits);
                 break;
             case OPERATION.OR:
                 bits = c.Or(bits);
                 break;
             case OPERATION.ANDNOT:
                 bits = c.And(bits.Not(maxsize));
                 break;
         }
     }
     else
         bits = c;
     return bits;
 }
Пример #19
0
        protected override Expression VisitBinary(BinaryExpression b)
        {
            var m = this.Visit(b.Left);

            if (m == null) // VB.net sty;e linq for string compare
            {
                return(b.Right);
            }
            ExpressionType t = b.NodeType;

            if (t == ExpressionType.Equal || t == ExpressionType.NotEqual ||
                t == ExpressionType.LessThan || t == ExpressionType.LessThanOrEqual ||
                t == ExpressionType.GreaterThan || t == ExpressionType.GreaterThanOrEqual)
            {
                _stack.Push(b.NodeType);
            }


            this.Visit(b.Right);
            t = b.NodeType;
            if (t == ExpressionType.Equal || t == ExpressionType.NotEqual ||
                t == ExpressionType.LessThanOrEqual || t == ExpressionType.LessThan ||
                t == ExpressionType.GreaterThanOrEqual || t == ExpressionType.GreaterThan
                )
            {
                // binary expression
                object         lv  = _stack.Pop();
                ExpressionType lo  = (ExpressionType)_stack.Pop();
                object         ln  = _stack.Pop();
                RDBExpression  exp = RDBExpression.Equal;
                // FEATURE : add contains , between, startswith
                if (lo == ExpressionType.LessThan)
                {
                    exp = RDBExpression.Less;
                }
                else if (lo == ExpressionType.LessThanOrEqual)
                {
                    exp = RDBExpression.LessEqual;
                }
                else if (lo == ExpressionType.GreaterThan)
                {
                    exp = RDBExpression.Greater;
                }
                else if (lo == ExpressionType.GreaterThanOrEqual)
                {
                    exp = RDBExpression.GreaterEqual;
                }
                else if (lo == ExpressionType.NotEqual)
                {
                    exp = RDBExpression.NotEqual;
                }

                _bitmap.Push(qexpression("" + ln, exp, lv));
            }

            if (t == ExpressionType.And || t == ExpressionType.AndAlso ||
                t == ExpressionType.Or || t == ExpressionType.OrElse)
            {
                if (_bitmap.Count > 1)
                {
                    // do bitmap operations
                    WAHBitArray r = (WAHBitArray)_bitmap.Pop();
                    WAHBitArray l = (WAHBitArray)_bitmap.Pop();

                    if (t == ExpressionType.And || t == ExpressionType.AndAlso)
                    {
                        _bitmap.Push(r.And(l));
                    }
                    if (t == ExpressionType.Or || t == ExpressionType.OrElse)
                    {
                        _bitmap.Push(r.Or(l));
                    }
                }
                else
                {
                    // single bitmap operation
                }
            }
            return(b);
        }
Пример #20
0
        public Mem(String mFile, uint[] BitmapArray = null, MemoryDescriptor Override = null)
        {
            GapScanSize   = 0x10000000;
            StartOfMemory = Override != null ? Override.StartOfMemmory : 0;

            MapViewBase = 0;
            MapViewSize = MapWindowSize = (0x1000 * 0x1000 * 4); // Due to the physical page allocation algorithm a modest window is probably fine

            // so not even 1/2 the size of the window which was only getting < 50% hit ratio at best
            // PageCache may be better off than a huge window...
            if (PageCache == null)
            {
                PageCache = new ConcurrentDictionary <long, long[]>(8, PageCacheMax);
            }

            if (BitmapArray != null)
            {
                pfnTableIdx = new WAHBitArray(WAHBitArray.TYPE.Bitarray, BitmapArray);
            }
            else
            {
                pfnTableIdx = new WAHBitArray();
            }

            // 32bit's of pages should be plenty?
            pfnTableIdx.Length = (int)(MapViewSize / 0x1000);

            if (File.Exists(mFile))
            {
                MemoryDump = mFile;
                FileSize   = new FileInfo(MemoryDump).Length;

                if (Override != null)
                {
                    MD = Override;
                }
                else
                {
                    MD = new MemoryDescriptor(FileSize);
                    if (DetectedDescriptor != null)
                    {
                        MD = DetectedDescriptor;
                    }
                }
                MapViewSize = FileSize < MapWindowSize ? FileSize : MapWindowSize;

                var lmindex = Interlocked.Increment(ref mindex);

                var mapName = Path.GetFileNameWithoutExtension(MemoryDump) + lmindex.ToString();

                mapStream  = new FileStream(MemoryDump, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
                mappedFile = MemoryMappedFile.CreateFromFile(mapStream,
                                                             mapName,
                                                             0,
                                                             MemoryMappedFileAccess.Read,
                                                             null,
                                                             HandleInheritability.Inheritable,
                                                             false);

                mappedAccess = mappedFile.CreateViewAccessor(
                    MapViewBase,
                    MapViewSize,
                    MemoryMappedFileAccess.Read);

                DiscoveredGaps = new Dictionary <long, long>();
            }
        }
 public QueryPlanResult(WAHBitArray bitArray, IEnumerable <KeyValuePair <DocumentIndexTypeToken, int> > versions)
 {
     _bitArray = bitArray;
     _versions = versions.ToDictionary(p => p.Key, p => p.Value);
 }
Пример #22
0
        private Result <T> ReturnRows2 <T>(WAHBitArray ba, List <T> trows, int start, int count, List <int> orderby)
        {
            DateTime   dt   = FastDateTime.Now;
            List <T>   rows = new List <T>();
            Result <T> ret  = new Result <T>();
            int        skip = start;
            int        c    = 0;

            ret.TotalCount = (int)ba.CountOnes();
            if (count == -1)
            {
                count = ret.TotalCount;
            }
            if (count > 0)
            {
                foreach (int i in orderby)
                {
                    if (ba.Get(i))
                    {
                        if (skip > 0)
                        {
                            skip--;
                        }
                        else
                        {
                            bool b = OutputRow <T>(rows, i);
                            if (b && count > 0)
                            {
                                c++;
                            }
                        }
                        ba.Set(i, false);
                        if (c == count)
                        {
                            break;
                        }
                    }
                }
                foreach (int i in ba.GetBitIndexes())
                {
                    if (c < count)
                    {
                        if (skip > 0)
                        {
                            skip--;
                        }
                        else
                        {
                            bool b = OutputRow <T>(rows, i);
                            if (b && count > 0)
                            {
                                c++;
                            }
                        }
                        if (c == count)
                        {
                            break;
                        }
                    }
                }
            }
            if (trows != null)// TODO : move to start and decrement in count
            {
                foreach (var o in trows)
                {
                    rows.Add(o);
                }
            }
            _log.Debug("query rows fetched (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds);
            _log.Debug("query rows count : " + rows.Count.ToString("#,0"));
            ret.OK    = true;
            ret.Count = rows.Count;
            ret.Rows  = rows;
            return(ret);
        }