Esempio n. 1
0
        public void Fun99()
        {
            long nid = 0;
            List <GubaBandResultEntity> list = null;

            Console.WriteLine("开始读数据");
            while ((list = DataContextMoudelFactory <GubaBandResultEntity> .GetDataContext("ConndbDB$GubaData").WhereBiger(p => p.ID, nid).Top(10000).OrderBy(p => p.ID).ExecuteList()).Count > 0)
            {
                nid = list.Last().ID;
                int      findcount = 0;
                DateTime now       = DateTime.Now;
                ProcessTraceUtil.StartTrace();
                foreach (var item in list)
                {
                    var re = BigEntityTableEngine.LocalEngine.Find <GubaBandResultEntity>("GubaBandResultEntity", "GubaCode_Uid_Recount", new object[] { item.GubaCode, item.Uid, item.Recount });
                    if (re.Count() == 0)
                    {
                        Console.WriteLine("找不到数据:" + item.GubaCode + "," + item.Uid + "," + item.Recount);
                        Console.Read();
                    }
                    else
                    {
                        findcount++;
                    }
                }
                LJC.FrameWork.LogManager.LogHelper.Instance.Debug("查找:" + ProcessTraceUtil.PrintTrace());
                Console.WriteLine("找到数:" + findcount + ",用时" + (DateTime.Now.Subtract(now).TotalMilliseconds) + "ms");
            }
            Console.WriteLine("读数据完成");
        }
Esempio n. 2
0
        public void textCount()
        {
            Console.WriteLine("输入key");
            string key = Console.ReadLine();
            var    now = DateTime.Now;

            ProcessTraceUtil.StartTrace();
            var total = BigEntityTableEngine.LocalEngine.Count("NewsKeysEntity");

            ProcessTraceUtil.Trace("total:" + total);
            ProcessTraceUtil.Trace("start find");
            var count = BigEntityTableEngine.LocalEngine.Count("NewsKeysEntity", "Keys", new object[] { key });

            Console.WriteLine("完成,用时:" + (DateTime.Now.Subtract(now).TotalMilliseconds + "ms,条数:" + count + "," + ProcessTraceUtil.PrintTrace()));
        }
Esempio n. 3
0
        private void TestFindKeyWord()
        {
            Console.WriteLine("输入key");
            string key = Console.ReadLine();
            var    now = DateTime.Now;

            ProcessTraceUtil.StartTrace();
            ProcessTraceUtil.Trace("start find");
            var count = BigEntityTableEngine.LocalEngine.Find <NewsKeysEntity>("NewsKeysEntity", "Keys", new[] { key }).Count();

            var detail = ProcessTraceUtil.PrintTrace();

            Console.WriteLine(detail);
            LJC.FrameWork.LogManager.LogHelper.Instance.Debug(detail);
        }
        public void MergeKey2(string tablename, string indexname, BigEntityTableMeta meta)
        {
            ProcessTraceUtil.StartTrace();

            IndexMergeInfo mergeinfo = null;

            lock (meta)
            {
                mergeinfo = meta.IndexMergeInfos.Find(p => indexname.Equals(p.IndexName));
                if (mergeinfo == null)
                {
                    mergeinfo           = new IndexMergeInfo();
                    mergeinfo.IndexName = indexname;
                    meta.IndexMergeInfos.Add(mergeinfo);
                }

                if (mergeinfo.IsMergin)
                {
                    return;
                }
                meta.NewAddCount   = 0;
                mergeinfo.IsMergin = true;
            }
            DateTime timestart    = DateTime.Now;
            string   newindexfile = string.Empty;

            byte[] bigbuffer   = new byte[1024 * 1024];
            byte[] smallbuffer = new byte[2048];
            try
            {
                ProcessTraceUtil.TraceMem("开始整理key:" + indexname, "m");

                long   lasmargepos      = 0;
                long   newIndexMergePos = mergeinfo.IndexMergePos;
                string indexfile        = indexname.Equals(meta.KeyName) ? GetKeyFile(tablename) : GetIndexFile(tablename, indexname);
                var    tablelocker      = GetKeyLocker(tablename, string.Empty);

                //新的磁盘索引
                List <BigEntityTableIndexItem> newdiskindexlist = new List <BigEntityTableIndexItem>();
                var loadFactor = (int)Math.Max(4, new FileInfo(indexfile).Length / MAX_KEYBUFFER);

                using (var reader = ObjTextReader.CreateReader(indexfile))
                {
                    long readoldstartpostion = reader.ReadedPostion();

                    try
                    {
                        tablelocker.EnterWriteLock();
                        keyindexmemtemplist[tablename] = keyindexmemlist[tablename];
                        keyindexmemlist[tablename]     = new SortArrayList <BigEntityTableIndexItem>();
                    }
                    finally
                    {
                        tablelocker.ExitWriteLock();
                    }
                    var listtemp = keyindexmemtemplist[tablename].GetList().ToList();
                    listtemp = listtemp.Select(p => new BigEntityTableIndexItem
                    {
                        Del        = p.Del,
                        Key        = p.Key,
                        KeyOffset  = p.KeyOffset,
                        len        = p.len,
                        Offset     = p.Offset,
                        Index      = p.Index,
                        RangeIndex = p.RangeIndex
                    }).ToList();
                    int readcount = listtemp.Count;

                    ProcessTraceUtil.TraceMem("从内存中读取新增数据,共" + readcount + "条", "m");

                    if (readcount == 0)
                    {
                        return;
                    }

                    lasmargepos = listtemp.Max(p => p.KeyOffset);
                    reader.SetPostion(lasmargepos);
                    reader.ReadObject <BigEntityTableIndexItem>();
                    lasmargepos = reader.ReadedPostion();

                    //优化确定哪些部分是不需要一个个读入的
                    long copybefore = 0, copylast = 0;
                    long lastrankindex = 0;
                    var  indexarray    = keyindexdisklist[tablename];
                    using (var sortarray = new Collections.SorteArray <BigEntityTableIndexItem>(indexarray))
                    {
                        int mid = -1;
                        int pos = sortarray.Find(listtemp.First(), ref mid);
                        ProcessTraceUtil.Trace("查找新数据在老数中插入的开始位置:mid=" + mid + ",pos=" + pos);
                        if (pos == -1 && mid != -1)
                        {
                            //小于最小的
                            copybefore    = indexarray[mid].KeyOffset;
                            lastrankindex = indexarray[mid].RangeIndex;
                            ProcessTraceUtil.Trace("老数据可以直接copy的部分:0->" + copybefore);
                        }
                        else if (pos != -1)
                        {
                            copybefore    = indexarray[pos].KeyOffset;
                            lastrankindex = indexarray[pos].RangeIndex;
                            ProcessTraceUtil.Trace("老数据可以直接copy的部分:0->" + copybefore);
                        }

                        //优化确定后面读到哪

                        mid = -1;
                        pos = sortarray.Find(listtemp.Last(), ref mid);
                        ProcessTraceUtil.Trace("查找新数据在老数据中插入的结束位置:mid=" + mid + ",pos=" + pos);
                        if (pos == -1 && mid != -1 && mid < indexarray.Length - 1)
                        {
                            //小于最小的
                            copylast = indexarray[mid + 1].KeyOffset;
                            ProcessTraceUtil.Trace("老数据可以直接copy的部分:" + copylast + "->" + mergeinfo.IndexMergePos);
                        }
                        else if (pos != -1)
                        {
                            copylast = indexarray[pos].KeyOffset;
                            ProcessTraceUtil.Trace("老数据可以直接copy的部分:" + copylast + "->" + mergeinfo.IndexMergePos);
                        }
                    }

                    newindexfile = (indexname.Equals(meta.KeyName) ? GetKeyFile(tablename) : GetIndexFile(tablename, indexname)) + ".temp";
                    if (File.Exists(newindexfile))
                    {
                        File.Delete(newindexfile);
                    }
                    //快速copy
                    if (copybefore > 0)
                    {
                        ProcessTraceUtil.TraceMem("直接copy前面不在排序范围的数据:0->" + copybefore, "m");
                        IOUtil.CopyFile(indexfile, newindexfile, FileMode.Create, 0, copybefore - 1);
                        readoldstartpostion = copybefore;
                        ProcessTraceUtil.TraceMem("copy数据完成", "m");

                        newdiskindexlist.AddRange(keyindexdisklist[tablename].Where(p => p.KeyOffset < copybefore));
                    }

                    bool isall = false;
                    ProcessTraceUtil.TraceMem("开始读取在排序范围内的数据", "m");
                    while (true)
                    {
                        ProcessTraceUtil.Trace("读取老数据,开始位置:" + readoldstartpostion);
                        reader.SetPostion(readoldstartpostion);
                        var  listordered = new List <BigEntityTableIndexItem>();
                        var  loadcount   = 0;
                        long keyoffset   = 0;
                        foreach (var item in reader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, bigbuffer))
                        {
                            item.KeyOffset = keyoffset;
                            item.SetIndex(meta.KeyIndexInfo);

                            if (item.KeyOffset >= mergeinfo.IndexMergePos)
                            {
                                break;
                            }
                            if (copylast > 0 && item.KeyOffset >= copylast)
                            {
                                break;
                            }
                            listordered.Add(item);
                            if (++loadcount >= MERGE_TRIGGER_NEW_COUNT)
                            {
                                break;
                            }
                        }

                        readoldstartpostion = reader.ReadedPostion();
                        bool isonlyoldlist = false;

                        if (listordered.Count == 0)
                        {
                            ProcessTraceUtil.TraceMem("老数据没有了,全部是新数据:" + listtemp.Count, "m");

                            listordered = MergeAndSort2(listordered, listtemp).ToList();
                            foreach (var item in listordered)
                            {
                                item.RangeIndex = lastrankindex++;
                            }
                            isall = true;
                        }
                        else if (listtemp.Count == 0 && listordered.Count > 10000)
                        {
                            ProcessTraceUtil.TraceMem("新数据没有了,全部是老数据:" + listordered.Count, "m");
                            //copy
                            IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, listordered.First().KeyOffset, listordered.Last().KeyOffset - 1, false);
                            var  listorderedlast = listordered.Last();
                            long copyoffset      = 0;
                            using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
                            {
                                var item = new BigEntityTableIndexItem {
                                    Del = listorderedlast.Del, Key = listorderedlast.Key, len = listorderedlast.len, Offset = listorderedlast.Offset, Index = listorderedlast.Index
                                };
                                item.KeyOffset = nw.GetWritePosition();

                                copyoffset = nw.GetWritePosition() - listorderedlast.KeyOffset;

                                nw.AppendObject(item);
                                newIndexMergePos = nw.GetWritePosition();
                            }
                            isonlyoldlist = true;
                            //更新索引
                            foreach (var item in listordered)
                            {
                                item.RangeIndex = lastrankindex++;
                                item.KeyOffset += copyoffset;
                            }

                            ProcessTraceUtil.TraceMem("直接copy数据完成", "m");
                        }
                        else
                        {
                            ProcessTraceUtil.TraceMem("老数据条数为:" + listordered.Count + ",新数据条数为:" + listtemp.Count, "m");

                            var listordermax = listordered.Last();

                            int idx = 0;
                            foreach (var item in listtemp)
                            {
                                if (item.CompareTo(listordermax) > 0)
                                {
                                    break;
                                }
                                else
                                {
                                    idx++;
                                }
                            }
                            List <BigEntityTableIndexItem> smalllist = listtemp.Take(idx).ToList();
                            listtemp    = listtemp.Skip(idx).ToList();
                            listordered = MergeAndSort2(listordered, smalllist).ToList();
                            foreach (var item in listordered)
                            {
                                item.RangeIndex = lastrankindex++;

                                //ProcessTraceUtil.Trace("rangeindex:" + item.Key[0] + "->" + (item.RangeIndex));
                            }
                            ProcessTraceUtil.TraceMem("排序完成:" + listordered.Count + "条", "m");
                        }

                        if (listordered.Count > 0)
                        {
                            if (!isonlyoldlist)
                            {
                                ProcessTraceUtil.TraceMem("把排好的数据写入到新索引文件:" + listordered.Count + "条", "m");
                                using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
                                {
                                    foreach (var item in listordered)
                                    {
                                        item.KeyOffset = nw.GetWritePosition();
                                        nw.AppendObject(item);
                                    }
                                    newIndexMergePos = nw.GetWritePosition();
                                }
                                ProcessTraceUtil.TraceMem("写入到新索引文件完成", "m");
                            }

                            if (listordered.Count <= 2 || loadFactor == 1)
                            {
                                newdiskindexlist.AddRange(listordered);
                            }
                            else
                            {
                                newdiskindexlist.Add(listordered.First());
                                int idx = 0;
                                foreach (var item in listordered)
                                {
                                    if ((++idx) % loadFactor == 0)
                                    {
                                        newdiskindexlist.Add(item);
                                    }
                                }
                                //newdiskindexlist.AddRange(listordered.Where(p => (++idx) % loadFactor == 0));
                                if (idx % loadFactor != 0)
                                {
                                    newdiskindexlist.Add(listordered.Last());
                                }
                            }
                        }

                        ProcessTraceUtil.TraceMem("写入到新索引文件后整理索引完成", "m");

                        if (isall)
                        {
                            if (copylast > 0 && copylast < mergeinfo.IndexMergePos)
                            {
                                ProcessTraceUtil.TraceMem("copy已排序的大于新增最大数部分" + copylast + "->" + mergeinfo.IndexMergePos, "m");
                                var offset = 0L;
                                using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
                                {
                                    offset = nw.GetWritePosition() - copylast;
                                }
                                //copy
                                long newindexpos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, copylast, mergeinfo.IndexMergePos - 1, false);

                                foreach (var p in keyindexdisklist[tablename])
                                {
                                    if (p.KeyOffset >= copylast && p.KeyOffset < mergeinfo.IndexMergePos)
                                    {
                                        newdiskindexlist.Add(new BigEntityTableIndexItem
                                        {
                                            Del        = p.Del,
                                            Key        = p.Key,
                                            KeyOffset  = p.KeyOffset + offset,
                                            len        = p.len,
                                            Offset     = p.Offset,
                                            Index      = p.Index,
                                            RangeIndex = p.RangeIndex + readcount
                                        });
                                    }
                                }

                                ProcessTraceUtil.TraceMem("copy数据完成->" + offset, "m");
                            }
                            break;
                        }
                        else
                        {
                            if (listtemp.Count > 0)
                            {
                                long newcopybefore = 0;
                                using (var sortarray = new Collections.SorteArray <BigEntityTableIndexItem>(indexarray))
                                {
                                    int mid = -1;
                                    int pos = sortarray.Find(listtemp.First(), ref mid);
                                    ProcessTraceUtil.Trace("查找已经排序的小于新增最小数据部分:mid=" + mid + ",pos=" + pos);
                                    if (pos == -1 && mid != -1)
                                    {
                                        //小于最小的
                                        newcopybefore = indexarray[mid].KeyOffset;
                                        lastrankindex = indexarray[mid].RangeIndex + readcount - listtemp.Count;
                                    }
                                    else if (pos != -1)
                                    {
                                        newcopybefore = indexarray[pos].KeyOffset;
                                        lastrankindex = indexarray[pos].RangeIndex + readcount - listtemp.Count;
                                    }
                                }
                                if (newcopybefore > readoldstartpostion)
                                {
                                    ProcessTraceUtil.Trace("中间copy");
                                    var offset = 0L;
                                    using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
                                    {
                                        offset = nw.GetWritePosition() - readoldstartpostion;
                                    }
                                    IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, readoldstartpostion, newcopybefore - 1, false);

                                    foreach (var p in keyindexdisklist[tablename])
                                    {
                                        if (p.KeyOffset >= readoldstartpostion && p.KeyOffset < newcopybefore)
                                        {
                                            newdiskindexlist.Add(new BigEntityTableIndexItem
                                            {
                                                Del        = p.Del,
                                                Key        = p.Key,
                                                KeyOffset  = p.KeyOffset + offset,
                                                len        = p.len,
                                                Offset     = p.Offset,
                                                Index      = p.Index,
                                                RangeIndex = p.RangeIndex + readcount - listtemp.Count
                                            });
                                        }
                                    }

                                    readoldstartpostion = newcopybefore;
                                    ProcessTraceUtil.Trace("中间copy完成");
                                }
                                else if (newcopybefore < readoldstartpostion)
                                {
                                    ProcessTraceUtil.Trace("补中间");
                                    reader.SetPostion(newcopybefore);
                                    //补充数目
                                    foreach (var item in reader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, smallbuffer))
                                    {
                                        item.KeyOffset = keyoffset;
                                        item.SetIndex(meta.KeyIndexInfo);
                                        //ProcessTraceUtil.Trace(item.Key[0].ToString());
                                        if (item.KeyOffset >= mergeinfo.IndexMergePos)
                                        {
                                            break;
                                        }
                                        if (copylast > 0 && item.KeyOffset >= copylast)
                                        {
                                            break;
                                        }
                                        if (item.KeyOffset == readoldstartpostion)
                                        {
                                            break;
                                        }
                                        if (item.Del)
                                        {
                                            continue;
                                        }
                                        //ProcessTraceUtil.Trace("补中间+1");
                                        lastrankindex++;
                                    }
                                }
                            }
                            else
                            {
                                ProcessTraceUtil.Trace("中间copy2");
                                var offset = 0L;
                                using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
                                {
                                    offset = nw.GetWritePosition() - readoldstartpostion;
                                }
                                IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, readoldstartpostion, copylast - 1, false);

                                foreach (var p in keyindexdisklist[tablename])
                                {
                                    if (p.KeyOffset >= readoldstartpostion && p.KeyOffset < copylast)
                                    {
                                        newdiskindexlist.Add(new BigEntityTableIndexItem
                                        {
                                            Del        = p.Del,
                                            Key        = p.Key,
                                            KeyOffset  = p.KeyOffset + offset,
                                            len        = p.len,
                                            Offset     = p.Offset,
                                            Index      = p.Index,
                                            RangeIndex = p.RangeIndex + readcount
                                        });
                                    }
                                }

                                readoldstartpostion = copylast;
                                ProcessTraceUtil.Trace("中间copy2完成");
                            }
                        }
                    }
                }

                //后面copy
                string tablefile = GetTableFile(tablename);

                var idxreader = ObjTextReader.CreateReader(indexfile);
                var newwriter = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2);
                try
                {
                    long nextcopypos = 0;
                    ProcessTraceUtil.TraceMem("读取后面的数据->" + lasmargepos, "m");
                    idxreader.SetPostion(lasmargepos);
                    bool hasitem   = false;
                    bool isfirst   = true;
                    long keyoffset = 0;
                    foreach (var item in idxreader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, smallbuffer))
                    {
                        hasitem = true;
                        item.SetIndex(meta.KeyIndexInfo);
                        item.KeyOffset = keyoffset;
                        if (item.KeyOffset > newwriter.GetWritePosition())
                        {
                            var spacelen = item.KeyOffset - newwriter.GetWritePosition();
                            ProcessTraceUtil.Trace("没有对齐,尝试对齐,spacelen->" + spacelen);
                            if (spacelen % 3 == 0)
                            {
                                newwriter.FillSpace(spacelen / 3);
                            }
                            else
                            {
                                ProcessTraceUtil.Trace("对齐失败");
                                var ex = new Exception("无法对齐");
                                ex.Data.Add("老索引文件当前位置", item.KeyOffset);
                                ex.Data.Add("新索引文件写入位置", newwriter.GetWritePosition());
                                throw ex;
                            }
                        }

                        if (item.KeyOffset == newwriter.GetWritePosition())
                        {
                            nextcopypos = newwriter.GetWritePosition();
                            if (isfirst)
                            {
                                newIndexMergePos = nextcopypos;
                            }
                            ProcessTraceUtil.Trace("新老索引文件已经对齐:" + item.KeyOffset);
                            break;
                        }

                        isfirst        = false;
                        item.KeyOffset = newwriter.GetWritePosition();
                        newwriter.AppendObject(item);
                    }

                    if (nextcopypos > 0)
                    {
                        newwriter.Dispose();
                        ProcessTraceUtil.Trace("copy后面的数据->" + nextcopypos);
                        nextcopypos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, nextcopypos, -512);
                    }
                    else if (!hasitem)
                    {
                        var idxpos = idxreader.ReadedPostion();
                        if (idxpos == newwriter.GetWritePosition())
                        {
                            nextcopypos = idxpos;
                            newwriter.Dispose();
                        }
                        else
                        {
                            ProcessTraceUtil.Trace(idxpos + " vs " + newwriter.GetWritePosition());
                        }
                    }
                    ProcessTraceUtil.TraceMem("读取后面的数据完成", "m");
                    if (nextcopypos == 0)
                    {
                        throw new Exception("更新索引出错");
                    }

                    try
                    {
                        BigEntityTableIndexItem[] oldindexarray     = null;
                        BigEntityTableIndexItem[] newdiskindexarray = newdiskindexlist.ToArray();

                        tablelocker.EnterWriteLock();
                        ProcessTraceUtil.TraceMem("读取后面的数据->" + lasmargepos, "m");
                        if (nextcopypos <= 0)
                        {
                            using (newwriter)
                            {
                                using (idxreader)
                                {
                                    foreach (var item in idxreader.ReadObjectsWating <BigEntityTableIndexItem>(1, bytes: bigbuffer))
                                    {
                                        item.SetIndex(meta.KeyIndexInfo);
                                        item.KeyOffset = newwriter.GetWritePosition();
                                        newwriter.AppendObject(item);
                                    }
                                }
                            }
                        }
                        else
                        {
                            nextcopypos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, nextcopypos, long.MaxValue);
                            ProcessTraceUtil.TraceMem("继续copy后面的数据->" + nextcopypos, "m");
                            idxreader.Dispose();
                        }
                        ProcessTraceUtil.TraceMem("读取后面的数据完成", "m");

                        //更新索引
                        mergeinfo.LoadFactor = loadFactor;
                        keyindexdisklist.TryRemove(tablename, out oldindexarray);
                        keyindexdisklist.TryAdd(tablename, newdiskindexarray);
                        keyindexmemtemplist[tablename] = new SortArrayList <BigEntityTableIndexItem>();

                        int trycount = 0;
                        while (true)
                        {
                            try
                            {
                                File.Delete(indexfile);
                                ProcessTraceUtil.Trace("删除旧文件完成");
                                break;
                            }
                            catch (System.IO.IOException ex)
                            {
                                Thread.Sleep(1);
                                trycount++;
                                if (trycount > 1000)
                                {
                                    throw ex;
                                }
                            }
                        }

                        File.Move(newindexfile, indexfile);

                        ProcessTraceUtil.TraceMem("删除旧文件,重命名新文件完成", "m");

                        idxreader = null;
                    }
                    finally
                    {
                        tablelocker.ExitWriteLock();
                    }


                    string metafile = GetMetaFile(tablename);
                    mergeinfo.IndexMergePos = lasmargepos;
                    SerializerHelper.SerializerToXML(meta, metafile, true);

                    ProcessTraceUtil.Trace("更新元文件,更新索引完成");
                }
                catch (Exception ex)
                {
                    Console.WriteLine("整理索引后面部分出错" + ex.ToString());

                    //LogManager.LogHelper.Instance.Error("整理索引后面部分出错", ex);
                }
                finally
                {
                    if (idxreader != null)
                    {
                        idxreader.Dispose();
                    }

                    if (newwriter != null && !newwriter.Isdispose)
                    {
                        newwriter.Dispose();
                    }

                    if (File.Exists(newindexfile))
                    {
                        File.Delete(newindexfile);
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());

                //LogManager.LogHelper.Instance.Error("整理索引出错", ex);
            }
            finally
            {
                //GC.Collect();
                //ProcessTraceUtil.TraceMem("回收内存","m");
                mergeinfo.IsMergin = false;
                var info = ProcessTraceUtil.PrintTrace();
                Console.WriteLine(info);
                //LogManager.LogHelper.Instance.Debug("整理索引过程:" + info);
            }
        }
Esempio n. 5
0
        public void AssertFindEqual <T>(string tablename) where T : new()
        {
            var meta = GetMetaData(tablename);
            List <Dictionary <object, int> > list = LJC.FrameWork.Comm.LocalCacheManager <List <Dictionary <object, int> > > .Find("asfasfaefaes", () =>
            {
                List <Dictionary <object, int> > list1 = new System.Collections.Generic.List <Dictionary <object, int> >();

                List <T> listt = new System.Collections.Generic.List <T>();
                foreach (var item in LocalEngine.Find <T>(tablename, (p) => true))
                {
                    listt.Add(item);
                }

                foreach (var idx in meta.IndexInfos)
                {
                    Dictionary <object, int> keyscount = new Dictionary <object, int>();
                    foreach (var item in listt)
                    {
                        var idxval = string.Join("_", idx.GetIndexValues(item, meta).Select(p => p.ToString()));
                        if (keyscount.ContainsKey(idxval))
                        {
                            keyscount[idxval] = keyscount[idxval] + 1;
                        }
                        else
                        {
                            keyscount.Add(idxval, 1);
                        }
                    }
                    list1.Add(keyscount);
                }

                return(list1);
            }, 36000);

            Console.WriteLine("分析完成");

            int      i   = 0;
            int      cnt = 0;
            DateTime now = DateTime.Now;

            //var thekey = "新疆城建";
            foreach (var dic in list)
            {
                foreach (var kv in dic)
                {
                    cnt++;

                    //if (!kv.Key.Equals(thekey))
                    //{
                    //    continue;
                    //}

                    ProcessTraceUtil.StartTrace();

                    var count = FindIndex(tablename, meta, meta.IndexInfos[i], new object[] { kv.Key }).Count();
                    if (count != kv.Value)
                    {
                        //Console.WriteLine("查找不相等:" + kv.Key + ",查找到的" + count + "!=统计的" + kv.Value);
                        LogManager.LogHelper.Instance.Error("[" + meta.IndexInfos[i].IndexName + "]查找不相等:" + kv.Key + ",查找到的" + count + "!=统计的" + kv.Value);
                    }

                    //Console.WriteLine(ProcessTraceUtil.PrintTrace());
                    if (cnt % 10000 == 0)
                    {
                        Console.WriteLine("10000条用时:" + (DateTime.Now.Subtract(now)).TotalMilliseconds + "ms");
                        now = DateTime.Now;
                    }
                }
                i++;
            }
        }
Esempio n. 6
0
        public static void TestScan(List <PersonInfo> memlist, int psstart = 1, int psend = 100)
        {
            int pi = 1, ps = psstart;

            memlist = memlist.OrderBy(p => p.Name).ToList();
            while (true)
            {
                pi = 1;
                while (true)
                {
                    ProcessTraceUtil.StartTrace();
                    ProcessTraceUtil.TraceMem("第" + pi + "页");
                    long total    = 0;
                    var  pagelist = memlist.Skip((pi - 1) * ps).Take(ps).ToList();
                    ProcessTraceUtil.TraceMem("内存排序完成");
                    var pagelist2 = BigEntityTableEngine.LocalEngine.Scan <PersonInfo>("PersonInfo", "Name_1", new object[] { string.Empty }, new object[] { "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" },
                                                                                       pi, ps, ref total);
                    ProcessTraceUtil.TraceMem("本地库排序完成");
                    if (total != memlist.Count)
                    {
                        throw new Exception("scan返回的总数错误");
                    }

                    if (pagelist.Count != pagelist2.Count)
                    {
                        throw new Exception("测试分页失败,第" + pi + "页数据不同,页面大小:" + ps);
                    }

                    for (int i = 0; i < pagelist.Count; i++)
                    {
                        if (pagelist[i].ID != pagelist2[i].ID)
                        {
                            throw new Exception("测试分页失败,第" + pi + "页数据不同,页面大小:" + ps);
                        }
                    }

                    if (pagelist.Count == 0)
                    {
                        break;
                    }

                    pi++;

                    ProcessTraceUtil.TraceMem("比较完成");

                    var log = ProcessTraceUtil.PrintTrace();
                    Console.WriteLine(pi + "->" + log);
                    if (!string.IsNullOrWhiteSpace(log))
                    {
                        LogHelper.Instance.Debug(log);
                    }
                }
                ps++;

                if (ps > psend)
                {
                    break;
                }
            }

            Console.WriteLine("TestScan测试完成,所有用例都通过");
        }
Esempio n. 7
0
        public static void Test()
        {
            //添加9999条数据
            var insertcount           = 9999;
            List <PersonInfo> memlist = new List <PersonInfo>();

            for (int i = 0; i < insertcount; i++)
            {
                var id = Guid.NewGuid().ToString();
                memlist.Add(new PersonInfo
                {
                    Name = "n" + id,
                    Addr = "addr",
                    Age  = new Random(id.GetHashCode()).Next(1, 101),
                    ID   = id,
                    Sex  = new Random(id.GetHashCode()).Next(0, 2)
                });
            }

            if (System.IO.Directory.Exists("localdb"))
            {
                System.IO.Directory.Delete("localdb", true);
            }

            BigEntityTableEngine.LocalEngine.CreateTable("PersonInfo", "ID", typeof(PersonInfo), new IndexInfo[]
            {
                new IndexInfo
                {
                    IndexName = "Name_1",
                    Indexs    = new IndexItem[] {
                        new IndexItem {
                            Direction = 1,
                            Field     = "Name",
                            FieldType = EntityType.STRING
                        }
                    }
                }
                , new IndexInfo
                {
                    IndexName = "Age-1",
                    Indexs    = new IndexItem[]
                    {
                        new IndexItem
                        {
                            Direction = -1,
                            Field     = "Age",
                            FieldType = EntityType.INT32
                        }
                    }
                }
            });

            BigEntityTableEngine.LocalEngine.InsertBatch <PersonInfo>("PersonInfo", memlist);
            var insertcount2 = BigEntityTableEngine.LocalEngine.Count("PersonInfo");

            if (insertcount != insertcount2)
            {
                throw new Exception("插入数据后统计数据测试不通过");
            }

            ////TestScan(memlist);

            //再写入1万条
            var insertmore = 1000;

            for (int k = 0; k < 330; k++)
            {
                var templist = new List <PersonInfo>();
                for (int i = 0; i < insertmore; i++)
                {
                    var id = Guid.NewGuid().ToString();
                    templist.Add(new PersonInfo
                    {
                        Name = "n" + id,
                        Addr = "addr",
                        Age  = new Random(id.GetHashCode()).Next(1, 101),
                        ID   = id,
                        Sex  = new Random(id.GetHashCode()).Next(0, 2)
                    });
                }

                ProcessTraceUtil.StartTrace();
                DateTime now = DateTime.Now;

                BigEntityTableEngine.LocalEngine.InsertBatch <PersonInfo>("PersonInfo", templist);
                LogHelper.Instance.Info(ProcessTraceUtil.PrintTrace());
                Console.WriteLine("写入万条数据完成:" + DateTime.Now.Subtract(now).TotalMilliseconds + "ms");
                memlist.AddRange(templist);
                //Thread.Sleep(1000);
            }

            Console.WriteLine("再次测试scan");
            //TestScan(memlist,999,999);
            TestAgeCount(memlist);
            TestTalbeCount(memlist);
            //TestScan2(memlist);
            //TestScan3(memlist);

            //删除
            //Console.WriteLine("测试删除");
            TestDel(ref memlist);
            TestMulDel(ref memlist);
            //TestScan2(memlist);
            //TestScan3(memlist);
            TestAgeCount(memlist);
            TestTalbeCount(memlist);

            List <PersonInfo> newpersonlist = new List <PersonInfo>();

            for (int i = 0; i < 10000; i++)
            {
                var id = Guid.NewGuid().ToString();
                newpersonlist.Add(new PersonInfo
                {
                    Name = "n" + id,
                    Addr = "addr",
                    Age  = new Random(id.GetHashCode()).Next(1, 101),
                    ID   = id,
                    Sex  = new Random(id.GetHashCode()).Next(0, 2)
                });
            }
            memlist.AddRange(newpersonlist);
            BigEntityTableEngine.LocalEngine.InsertBatch <PersonInfo>("PersonInfo", newpersonlist);
            Console.WriteLine("插入一批数据");
            //TestScan(memlist, 999, 999);
            //TestScan2(memlist);
            //TestScan3(memlist);
            TestAgeCount(memlist);
            TestTalbeCount(memlist);

            Console.WriteLine("删除后新增数据测试成功");

            //测试更新
            TestUpdate(ref memlist);

            //TestScan2(memlist);
            //TestScan3(memlist);
            TestAgeCount(memlist);
            TestTalbeCount(memlist);

            BigEntityTableEngine.LocalEngine.UnLoadTable("PersonInfo");
            //TestScan2(memlist);
            //TestScan3(memlist);
            TestAgeCount(memlist);
            TestTalbeCount(memlist);
        }