Esempio n. 1
0
        static void TestRWObjEx()
        {
            int    i        = 0;
            string filename = "testrwobjex.bin";

            using (LJC.FrameWork.Comm.ObjTextWriter writer = ObjTextWriter.CreateWriter(filename, ObjTextReaderWriterEncodeType.protobufex))
            {
                while ((i++) < 100)
                {
                    writer.AppendObject <Man>(new Man
                    {
                        Name   = "李金川" + i,
                        IDCard = "421182198612301310",
                        Addr   = "湖北省武穴市",
                        Sex    = 1
                    });

                    //writer.Flush();

                    Console.WriteLine("写入成功一条!" + i);

                    Thread.Sleep(10);
                }
            }
        }
Esempio n. 2
0
        static void TestRWObj1()
        {
            int    i        = 0;
            string filename = "testrwobj1.bin";

            System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
            sw.Start();
            using (LJC.FrameWork.Comm.ObjTextWriter writer = ObjTextWriter.CreateWriter(filename, ObjTextReaderWriterEncodeType.protobufex))
            {
                while ((i++) < 1000000)
                {
                    writer.AppendObject <Man>(new Man
                    {
                        Name   = "李金川" + i,
                        IDCard = "421182198612301310",
                        Addr   = "湖北省武穴市",
                        Sex    = 1
                    });

                    //writer.Flush();

                    if (i % 10000 == 0)
                    {
                        Console.WriteLine("写入成功到" + i);
                    }

                    //Thread.Sleep(1000);
                }
            }
            sw.Stop();
            Console.WriteLine("共用时:" + sw.Elapsed.TotalSeconds + "秒");
        }
Esempio n. 3
0
        public bool Delete(string tablename, string key)
        {
            EntityTableMeta meta = GetMetaData(tablename);
            Dictionary <long, EntityTableIndexItem> arr = null;

            var locker = GetKeyLocker(tablename, string.Empty);

            lock (locker)
            {
                if (keyindexdic[tablename].TryRemove(key, out arr))
                {
                    string keyindexfile = GetKeyFile(tablename);
                    using (ObjTextWriter keywriter = ObjTextWriter.CreateWriter(keyindexfile, ObjTextReaderWriterEncodeType.entitybuf))
                    //ObjTextWriter keywriter = GetWriter(keyindexfile);
                    //lock(keywriter)
                    {
                        foreach (var item in arr)
                        {
                            var indexitem = (EntityTableIndexItem)item.Value;
                            indexitem.Del = true;
                            keywriter.AppendObject(indexitem);

                            foreach (var idx in meta.Indexs)
                            {
                                string indexfile = GetIndexFile(tablename, idx);
                                using (ObjTextWriter idxwriter = ObjTextWriter.CreateWriter(indexfile, ObjTextReaderWriterEncodeType.entitybuf))
                                //ObjTextWriter idxwriter = GetWriter(indexfile);
                                //lock(idxwriter)
                                {
                                    idxwriter.AppendObject(indexitem);
                                }
                            }
                        }
                    }

                    return(true);
                }
            }

            return(false);
        }
Esempio n. 4
0
        private ObjTextWriter GetWriter(string filename, int locksecs = 1)
        {
            ObjTextWriter writer = null;

            if (writerdic.TryGetValue(filename, out writer))
            {
                lock (writer)
                {
                    if (!writer.Isdispose)
                    {
                        writer.Tag = DateTime.Now;
                        return(writer);
                    }
                }
            }

            lock (writerdic)
            {
                if (writerdic.TryGetValue(filename, out writer))
                {
                    lock (writer)
                    {
                        if (!writer.Isdispose)
                        {
                            writer.Tag = DateTime.Now;
                            return(writer);
                        }
                    }
                }

                writer     = ObjTextWriter.CreateWriter(filename, ObjTextReaderWriterEncodeType.entitybuf);
                writer.Tag = DateTime.Now;
                writerdic.Add(filename, writer);

                LJC.FrameWork.Comm.Coroutine.CoroutineEngine.DefaultCoroutineEngine.Dispatcher(new WriterDestroy(writerdic, filename, locksecs));
            }
            return(writer);
        }
Esempio n. 5
0
        private bool Update2 <T>(string tablename, string key, T item, EntityTableMeta meta) where T : new()
        {
            string tablefile = GetTableFile(tablename);
            Dictionary <long, EntityTableIndexItem> arr = null;
            Tuple <long, long> offset = null;
            int indexpos = 0;

            var locker = GetKeyLocker(tablename, string.Empty);

            lock (locker)
            {
                if (!keyindexdic[tablename].TryGetValue(key, out arr))
                {
                    throw new Exception(string.Format("更新失败,key为{0}的记录数为0", key));
                }
                string keyindexfile            = GetKeyFile(tablename);
                EntityTableIndexItem indexitem = (EntityTableIndexItem)arr.Last().Value;
                using (ObjTextWriter keywriter = ObjTextWriter.CreateWriter(keyindexfile, ObjTextReaderWriterEncodeType.entitybuf))
                //ObjTextWriter keywriter = GetWriter(keyindexfile);
                {
                    indexitem.Del = true;
                    keywriter.AppendObject(indexitem);
                }

                var keyvalue = item.Eval(meta.KeyProperty);
                //var keyvalue=meta.KeyProperty.GetValueMethed(item);
                if (keyvalue == null)
                {
                    throw new Exception("key值不能为空");
                }
                var tableitem = new EntityTableItem <T>(item);
                tableitem.Flag = (byte)EntityTableItemFlag.Ok;
                using (ObjTextWriter otw = ObjTextWriter.CreateWriter(tablefile, ObjTextReaderWriterEncodeType.entitybuf))
                //ObjTextWriter otw = GetWriter(tablefile);
                {
                    offset = otw.PreAppendObject(tableitem, (s1, s2) =>
                    {
                        if (s1.Length <= indexitem.len)
                        {
                            Console.WriteLine("修改->覆盖");
                            return(otw.Override(indexitem.Offset, s1));
                        }
                        return(null);
                    });
                }


                Dictionary <long, EntityTableIndexItem> al = null;
                var keyidc = keyindexdic[tablename];
                if (!keyidc.TryGetValue(keyvalue.ToString(), out al))
                {
                    lock (keyidc)
                    {
                        if (!keyidc.TryGetValue(keyvalue.ToString(), out al))
                        {
                            al = new Dictionary <long, EntityTableIndexItem>();
                            keyidc.TryAdd(keyvalue.ToString(), al);
                        }
                    }
                }

                EntityTableIndexItem newkey = new EntityTableIndexItem
                {
                    Key    = keyvalue.ToString(),
                    Offset = offset.Item1,
                    len    = (int)(offset.Item2 - offset.Item1),
                    Del    = false
                };

                if (newkey.Offset == indexitem.Offset)
                {
                    newkey.len = indexitem.len;
                }

                arr[indexpos] = newkey;

                using (ObjTextWriter keyidxwriter = ObjTextWriter.CreateWriter(keyindexfile, ObjTextReaderWriterEncodeType.entitybuf))
                //ObjTextWriter keyidxwriter = GetWriter(keyindexfile);
                //lock(keyidxwriter)
                {
                    keyidxwriter.AppendObject(newkey);
                }

                foreach (var idx in meta.Indexs)
                {
                    string indexfile  = GetIndexFile(tablename, idx);
                    var    indexvalue = item.Eval(meta.IndexProperties[idx]);
                    var    newindex   = new EntityTableIndexItem
                    {
                        Key    = indexvalue == null ? string.Empty : indexvalue.ToString(),
                        Offset = offset.Item1,
                        len    = (int)(offset.Item2 - offset.Item1),
                        Del    = false
                    };
                    using (ObjTextWriter idxwriter = ObjTextWriter.CreateWriter(indexfile, ObjTextReaderWriterEncodeType.entitybuf))
                    //ObjTextWriter idxwriter = GetWriter(indexfile);
                    //lock(idxwriter)
                    {
                        idxwriter.AppendObject(newindex);
                    }
                }

                Console.WriteLine("写入成功:" + keyvalue + "->" + offset);
            }
            return(true);
        }
Esempio n. 6
0
        private bool Insert2 <T>(string tablename, IEnumerable <T> items, EntityTableMeta meta) where T : new()
        {
            string tablefile = GetTableFile(tablename);

            var locker = GetKeyLocker(tablename, string.Empty);

            lock (locker)
            {
                ObjTextWriter otw          = ObjTextWriter.CreateWriter(tablefile, ObjTextReaderWriterEncodeType.entitybuf);
                string        keyindexfile = GetKeyFile(tablename);
                ObjTextWriter keywriter    = ObjTextWriter.CreateWriter(keyindexfile, ObjTextReaderWriterEncodeType.entitybuf);
                Dictionary <string, ObjTextWriter> idxwriterdic = new Dictionary <string, ObjTextWriter>();

                try
                {
                    foreach (var item in items)
                    {
                        var keyvalue = item.Eval(meta.KeyProperty);
                        //var keyvalue = meta.KeyProperty.GetValueMethed(item);
                        if (keyvalue == null)
                        {
                            throw new Exception("key值不能为空");
                        }

                        var keystr = keyvalue.ToString();

                        var keylocker = GetKeyLocker(tablename, keystr);

                        Dictionary <long, EntityTableIndexItem> arr = null;
                        lock (keylocker)
                        {
                            if (keyindexdic[tablename].TryGetValue(keystr, out arr))
                            {
                                throw new Exception(string.Format("key:{0}不可重复", keystr));
                            }
                        }

                        var tableitem = new EntityTableItem <T>(item);
                        tableitem.Flag = (byte)EntityTableItemFlag.Ok;

                        //using (ObjTextWriter otw = ObjTextWriter.CreateWriter(tablefile, ObjTextReaderWriterEncodeType.entitybuf))
                        //var otw = GetWriter(tablefile);
                        //lock(otw)
                        {
                            var offset = otw.AppendObject(tableitem);

                            Dictionary <long, EntityTableIndexItem> al = null;
                            var idc = keyindexdic[tablename];
                            if (!idc.TryGetValue(keystr, out al))
                            {
                                lock (idc)
                                {
                                    if (!idc.TryGetValue(keystr, out al))
                                    {
                                        al = new Dictionary <long, EntityTableIndexItem>();
                                        idc.TryAdd(keystr, al);
                                    }
                                }
                            }

                            var newkey = new EntityTableIndexItem
                            {
                                Key    = keystr,
                                Offset = offset.Item1,
                                len    = (int)(offset.Item2 - offset.Item1)
                            };
                            lock (al)
                            {
                                al.Add(newkey.Offset, newkey);
                            }

                            //string keyindexfile = GetKeyFile(tablename);
                            //using (ObjTextWriter keywriter = ObjTextWriter.CreateWriter(keyindexfile, ObjTextReaderWriterEncodeType.entitybuf))
                            //ObjTextWriter keywriter = GetWriter(keyindexfile);
                            //lock(keywriter)
                            {
                                keywriter.AppendObject(newkey);
                            }

                            foreach (var idx in meta.Indexs)
                            {
                                string indexfile  = GetIndexFile(tablename, idx);
                                var    indexvalue = item.Eval(meta.IndexProperties[idx]);
                                var    newindex   = new EntityTableIndexItem
                                {
                                    Key    = indexvalue == null ? string.Empty : indexvalue.ToString(),
                                    Offset = offset.Item1,
                                    len    = (int)(offset.Item2 - offset.Item1)
                                };
                                //using (ObjTextWriter idxwriter = ObjTextWriter.CreateWriter(indexfile, ObjTextReaderWriterEncodeType.entitybuf))
                                //ObjTextWriter idxwriter = GetWriter(indexfile);
                                //lock(idxwriter)
                                {
                                    ObjTextWriter idxwriter = null;
                                    if (!idxwriterdic.TryGetValue(indexfile, out idxwriter))
                                    {
                                        idxwriter = ObjTextWriter.CreateWriter(indexfile, ObjTextReaderWriterEncodeType.entitybuf);
                                        idxwriterdic.Add(indexfile, idxwriter);
                                    }

                                    idxwriter.AppendObject(newindex);
                                }
                            }
                        }
                    }
                }
                finally
                {
                    otw.Dispose();
                    keywriter.Dispose();
                    foreach (var kv in idxwriterdic)
                    {
                        kv.Value.Dispose();
                    }
                }
            }

            return(true);
        }
Esempio n. 7
0
        public void CreateTable(string tablename, string keyname, Type ttype, string[] indexs = null)
        {
            string tablefile = GetTableFile(tablename);
            bool   delfile   = true;

            if (!File.Exists(tablefile))
            {
                try
                {
                    using (ObjTextWriter otw = ObjTextWriter.CreateWriter(tablefile, ObjTextReaderWriterEncodeType.entitybuf))
                    //ObjTextWriter otw = GetWriter(tablefile);
                    //lock(otw)
                    {
                        string metafile = GetMetaFile(tablename);
                        if (!File.Exists(metafile))
                        {
                            EntityTableMeta meta = new EntityTableMeta();
                            meta.KeyName = keyname;
                            meta.Indexs  = indexs ?? new string[] { };
                            meta.CTime   = DateTime.Now;
                            meta.TType   = ttype;
                            var pp = ReflectionHelper.GetProperty(ttype, keyname);
                            if (pp == null)
                            {
                                throw new Exception("找不到主键:" + keyname);
                            }
                            meta.KeyProperty = new PropertyInfoEx(pp);

                            if (indexs != null && indexs.Length > 0)
                            {
                                if (indexs.Contains(keyname))
                                {
                                    throw new Exception("索引不能包含主键");
                                }
                                indexs = indexs.Distinct().ToArray();
                                foreach (var idx in indexs)
                                {
                                    var idxpp = ReflectionHelper.GetProperty(ttype, idx);
                                    if (idxpp == null)
                                    {
                                        throw new Exception("找不到索引:" + idx);
                                    }

                                    if (!meta.IndexProperties.ContainsKey(idx))
                                    {
                                        meta.IndexProperties.Add(idx, new PropertyInfoEx(idxpp));
                                    }
                                }
                            }

                            LJC.FrameWork.Comm.SerializerHelper.SerializerToXML <EntityTableMeta>(meta, metafile, catchErr: true);
                            metadic.Add(tablename, meta);

                            keyindexdic.TryAdd(tablename, new ConcurrentDictionary <string, Dictionary <long, EntityTableIndexItem> >());
                        }
                        else
                        {
                            var meta = LJC.FrameWork.Comm.SerializerHelper.DeSerializerFile <EntityTableMeta>(metafile, true);
                            if (!meta.KeyName.Equals(keyname) || !meta.TypeString.Equals(ttype))
                            {
                                delfile = false;

                                throw new Exception("meta文件内容检查不一致");
                            }
                        }

                        //索引
                        string keyfile = GetKeyFile(tablename);
                        using (ObjTextWriter keyidxwriter = ObjTextWriter.CreateWriter(keyfile, ObjTextReaderWriterEncodeType.entitybuf))
                        //ObjTextWriter keyidxwriter = GetWriter(keyfile);
                        {
                        }

                        if (indexs != null)
                        {
                            foreach (var idx in indexs)
                            {
                                var indexfile = GetIndexFile(tablename, idx);
                                using (ObjTextWriter idxwriter = ObjTextWriter.CreateWriter(indexfile, ObjTextReaderWriterEncodeType.entitybuf))
                                //ObjTextWriter idxwriter = GetWriter(indexfile);
                                {
                                }
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    if (delfile && File.Exists(tablefile))
                    {
                        File.Delete(tablefile);
                    }
                    throw ex;
                }
            }
        }
        public void MergeKey2(string tablename, string indexname, BigEntityTableMeta meta)
        {
            ProcessTraceUtil.StartTrace();

            IndexMergeInfo mergeinfo = null;

            lock (meta)
            {
                mergeinfo = meta.IndexMergeInfos.Find(p => indexname.Equals(p.IndexName));
                if (mergeinfo == null)
                {
                    mergeinfo           = new IndexMergeInfo();
                    mergeinfo.IndexName = indexname;
                    meta.IndexMergeInfos.Add(mergeinfo);
                }

                if (mergeinfo.IsMergin)
                {
                    return;
                }
                meta.NewAddCount   = 0;
                mergeinfo.IsMergin = true;
            }
            DateTime timestart    = DateTime.Now;
            string   newindexfile = string.Empty;

            byte[] bigbuffer   = new byte[1024 * 1024];
            byte[] smallbuffer = new byte[2048];
            try
            {
                ProcessTraceUtil.TraceMem("开始整理key:" + indexname, "m");

                long   lasmargepos      = 0;
                long   newIndexMergePos = mergeinfo.IndexMergePos;
                string indexfile        = indexname.Equals(meta.KeyName) ? GetKeyFile(tablename) : GetIndexFile(tablename, indexname);
                var    tablelocker      = GetKeyLocker(tablename, string.Empty);

                //新的磁盘索引
                List <BigEntityTableIndexItem> newdiskindexlist = new List <BigEntityTableIndexItem>();
                var loadFactor = (int)Math.Max(4, new FileInfo(indexfile).Length / MAX_KEYBUFFER);

                using (var reader = ObjTextReader.CreateReader(indexfile))
                {
                    long readoldstartpostion = reader.ReadedPostion();

                    try
                    {
                        tablelocker.EnterWriteLock();
                        keyindexmemtemplist[tablename] = keyindexmemlist[tablename];
                        keyindexmemlist[tablename]     = new SortArrayList <BigEntityTableIndexItem>();
                    }
                    finally
                    {
                        tablelocker.ExitWriteLock();
                    }
                    var listtemp = keyindexmemtemplist[tablename].GetList().ToList();
                    listtemp = listtemp.Select(p => new BigEntityTableIndexItem
                    {
                        Del        = p.Del,
                        Key        = p.Key,
                        KeyOffset  = p.KeyOffset,
                        len        = p.len,
                        Offset     = p.Offset,
                        Index      = p.Index,
                        RangeIndex = p.RangeIndex
                    }).ToList();
                    int readcount = listtemp.Count;

                    ProcessTraceUtil.TraceMem("从内存中读取新增数据,共" + readcount + "条", "m");

                    if (readcount == 0)
                    {
                        return;
                    }

                    lasmargepos = listtemp.Max(p => p.KeyOffset);
                    reader.SetPostion(lasmargepos);
                    reader.ReadObject <BigEntityTableIndexItem>();
                    lasmargepos = reader.ReadedPostion();

                    //优化确定哪些部分是不需要一个个读入的
                    long copybefore = 0, copylast = 0;
                    long lastrankindex = 0;
                    var  indexarray    = keyindexdisklist[tablename];
                    using (var sortarray = new Collections.SorteArray <BigEntityTableIndexItem>(indexarray))
                    {
                        int mid = -1;
                        int pos = sortarray.Find(listtemp.First(), ref mid);
                        ProcessTraceUtil.Trace("查找新数据在老数中插入的开始位置:mid=" + mid + ",pos=" + pos);
                        if (pos == -1 && mid != -1)
                        {
                            //小于最小的
                            copybefore    = indexarray[mid].KeyOffset;
                            lastrankindex = indexarray[mid].RangeIndex;
                            ProcessTraceUtil.Trace("老数据可以直接copy的部分:0->" + copybefore);
                        }
                        else if (pos != -1)
                        {
                            copybefore    = indexarray[pos].KeyOffset;
                            lastrankindex = indexarray[pos].RangeIndex;
                            ProcessTraceUtil.Trace("老数据可以直接copy的部分:0->" + copybefore);
                        }

                        //优化确定后面读到哪

                        mid = -1;
                        pos = sortarray.Find(listtemp.Last(), ref mid);
                        ProcessTraceUtil.Trace("查找新数据在老数据中插入的结束位置:mid=" + mid + ",pos=" + pos);
                        if (pos == -1 && mid != -1 && mid < indexarray.Length - 1)
                        {
                            //小于最小的
                            copylast = indexarray[mid + 1].KeyOffset;
                            ProcessTraceUtil.Trace("老数据可以直接copy的部分:" + copylast + "->" + mergeinfo.IndexMergePos);
                        }
                        else if (pos != -1)
                        {
                            copylast = indexarray[pos].KeyOffset;
                            ProcessTraceUtil.Trace("老数据可以直接copy的部分:" + copylast + "->" + mergeinfo.IndexMergePos);
                        }
                    }

                    newindexfile = (indexname.Equals(meta.KeyName) ? GetKeyFile(tablename) : GetIndexFile(tablename, indexname)) + ".temp";
                    if (File.Exists(newindexfile))
                    {
                        File.Delete(newindexfile);
                    }
                    //快速copy
                    if (copybefore > 0)
                    {
                        ProcessTraceUtil.TraceMem("直接copy前面不在排序范围的数据:0->" + copybefore, "m");
                        IOUtil.CopyFile(indexfile, newindexfile, FileMode.Create, 0, copybefore - 1);
                        readoldstartpostion = copybefore;
                        ProcessTraceUtil.TraceMem("copy数据完成", "m");

                        newdiskindexlist.AddRange(keyindexdisklist[tablename].Where(p => p.KeyOffset < copybefore));
                    }

                    bool isall = false;
                    ProcessTraceUtil.TraceMem("开始读取在排序范围内的数据", "m");
                    while (true)
                    {
                        ProcessTraceUtil.Trace("读取老数据,开始位置:" + readoldstartpostion);
                        reader.SetPostion(readoldstartpostion);
                        var  listordered = new List <BigEntityTableIndexItem>();
                        var  loadcount   = 0;
                        long keyoffset   = 0;
                        foreach (var item in reader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, bigbuffer))
                        {
                            item.KeyOffset = keyoffset;
                            item.SetIndex(meta.KeyIndexInfo);

                            if (item.KeyOffset >= mergeinfo.IndexMergePos)
                            {
                                break;
                            }
                            if (copylast > 0 && item.KeyOffset >= copylast)
                            {
                                break;
                            }
                            listordered.Add(item);
                            if (++loadcount >= MERGE_TRIGGER_NEW_COUNT)
                            {
                                break;
                            }
                        }

                        readoldstartpostion = reader.ReadedPostion();
                        bool isonlyoldlist = false;

                        if (listordered.Count == 0)
                        {
                            ProcessTraceUtil.TraceMem("老数据没有了,全部是新数据:" + listtemp.Count, "m");

                            listordered = MergeAndSort2(listordered, listtemp).ToList();
                            foreach (var item in listordered)
                            {
                                item.RangeIndex = lastrankindex++;
                            }
                            isall = true;
                        }
                        else if (listtemp.Count == 0 && listordered.Count > 10000)
                        {
                            ProcessTraceUtil.TraceMem("新数据没有了,全部是老数据:" + listordered.Count, "m");
                            //copy
                            IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, listordered.First().KeyOffset, listordered.Last().KeyOffset - 1, false);
                            var  listorderedlast = listordered.Last();
                            long copyoffset      = 0;
                            using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
                            {
                                var item = new BigEntityTableIndexItem {
                                    Del = listorderedlast.Del, Key = listorderedlast.Key, len = listorderedlast.len, Offset = listorderedlast.Offset, Index = listorderedlast.Index
                                };
                                item.KeyOffset = nw.GetWritePosition();

                                copyoffset = nw.GetWritePosition() - listorderedlast.KeyOffset;

                                nw.AppendObject(item);
                                newIndexMergePos = nw.GetWritePosition();
                            }
                            isonlyoldlist = true;
                            //更新索引
                            foreach (var item in listordered)
                            {
                                item.RangeIndex = lastrankindex++;
                                item.KeyOffset += copyoffset;
                            }

                            ProcessTraceUtil.TraceMem("直接copy数据完成", "m");
                        }
                        else
                        {
                            ProcessTraceUtil.TraceMem("老数据条数为:" + listordered.Count + ",新数据条数为:" + listtemp.Count, "m");

                            var listordermax = listordered.Last();

                            int idx = 0;
                            foreach (var item in listtemp)
                            {
                                if (item.CompareTo(listordermax) > 0)
                                {
                                    break;
                                }
                                else
                                {
                                    idx++;
                                }
                            }
                            List <BigEntityTableIndexItem> smalllist = listtemp.Take(idx).ToList();
                            listtemp    = listtemp.Skip(idx).ToList();
                            listordered = MergeAndSort2(listordered, smalllist).ToList();
                            foreach (var item in listordered)
                            {
                                item.RangeIndex = lastrankindex++;

                                //ProcessTraceUtil.Trace("rangeindex:" + item.Key[0] + "->" + (item.RangeIndex));
                            }
                            ProcessTraceUtil.TraceMem("排序完成:" + listordered.Count + "条", "m");
                        }

                        if (listordered.Count > 0)
                        {
                            if (!isonlyoldlist)
                            {
                                ProcessTraceUtil.TraceMem("把排好的数据写入到新索引文件:" + listordered.Count + "条", "m");
                                using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
                                {
                                    foreach (var item in listordered)
                                    {
                                        item.KeyOffset = nw.GetWritePosition();
                                        nw.AppendObject(item);
                                    }
                                    newIndexMergePos = nw.GetWritePosition();
                                }
                                ProcessTraceUtil.TraceMem("写入到新索引文件完成", "m");
                            }

                            if (listordered.Count <= 2 || loadFactor == 1)
                            {
                                newdiskindexlist.AddRange(listordered);
                            }
                            else
                            {
                                newdiskindexlist.Add(listordered.First());
                                int idx = 0;
                                foreach (var item in listordered)
                                {
                                    if ((++idx) % loadFactor == 0)
                                    {
                                        newdiskindexlist.Add(item);
                                    }
                                }
                                //newdiskindexlist.AddRange(listordered.Where(p => (++idx) % loadFactor == 0));
                                if (idx % loadFactor != 0)
                                {
                                    newdiskindexlist.Add(listordered.Last());
                                }
                            }
                        }

                        ProcessTraceUtil.TraceMem("写入到新索引文件后整理索引完成", "m");

                        if (isall)
                        {
                            if (copylast > 0 && copylast < mergeinfo.IndexMergePos)
                            {
                                ProcessTraceUtil.TraceMem("copy已排序的大于新增最大数部分" + copylast + "->" + mergeinfo.IndexMergePos, "m");
                                var offset = 0L;
                                using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
                                {
                                    offset = nw.GetWritePosition() - copylast;
                                }
                                //copy
                                long newindexpos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, copylast, mergeinfo.IndexMergePos - 1, false);

                                foreach (var p in keyindexdisklist[tablename])
                                {
                                    if (p.KeyOffset >= copylast && p.KeyOffset < mergeinfo.IndexMergePos)
                                    {
                                        newdiskindexlist.Add(new BigEntityTableIndexItem
                                        {
                                            Del        = p.Del,
                                            Key        = p.Key,
                                            KeyOffset  = p.KeyOffset + offset,
                                            len        = p.len,
                                            Offset     = p.Offset,
                                            Index      = p.Index,
                                            RangeIndex = p.RangeIndex + readcount
                                        });
                                    }
                                }

                                ProcessTraceUtil.TraceMem("copy数据完成->" + offset, "m");
                            }
                            break;
                        }
                        else
                        {
                            if (listtemp.Count > 0)
                            {
                                long newcopybefore = 0;
                                using (var sortarray = new Collections.SorteArray <BigEntityTableIndexItem>(indexarray))
                                {
                                    int mid = -1;
                                    int pos = sortarray.Find(listtemp.First(), ref mid);
                                    ProcessTraceUtil.Trace("查找已经排序的小于新增最小数据部分:mid=" + mid + ",pos=" + pos);
                                    if (pos == -1 && mid != -1)
                                    {
                                        //小于最小的
                                        newcopybefore = indexarray[mid].KeyOffset;
                                        lastrankindex = indexarray[mid].RangeIndex + readcount - listtemp.Count;
                                    }
                                    else if (pos != -1)
                                    {
                                        newcopybefore = indexarray[pos].KeyOffset;
                                        lastrankindex = indexarray[pos].RangeIndex + readcount - listtemp.Count;
                                    }
                                }
                                if (newcopybefore > readoldstartpostion)
                                {
                                    ProcessTraceUtil.Trace("中间copy");
                                    var offset = 0L;
                                    using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
                                    {
                                        offset = nw.GetWritePosition() - readoldstartpostion;
                                    }
                                    IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, readoldstartpostion, newcopybefore - 1, false);

                                    foreach (var p in keyindexdisklist[tablename])
                                    {
                                        if (p.KeyOffset >= readoldstartpostion && p.KeyOffset < newcopybefore)
                                        {
                                            newdiskindexlist.Add(new BigEntityTableIndexItem
                                            {
                                                Del        = p.Del,
                                                Key        = p.Key,
                                                KeyOffset  = p.KeyOffset + offset,
                                                len        = p.len,
                                                Offset     = p.Offset,
                                                Index      = p.Index,
                                                RangeIndex = p.RangeIndex + readcount - listtemp.Count
                                            });
                                        }
                                    }

                                    readoldstartpostion = newcopybefore;
                                    ProcessTraceUtil.Trace("中间copy完成");
                                }
                                else if (newcopybefore < readoldstartpostion)
                                {
                                    ProcessTraceUtil.Trace("补中间");
                                    reader.SetPostion(newcopybefore);
                                    //补充数目
                                    foreach (var item in reader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, smallbuffer))
                                    {
                                        item.KeyOffset = keyoffset;
                                        item.SetIndex(meta.KeyIndexInfo);
                                        //ProcessTraceUtil.Trace(item.Key[0].ToString());
                                        if (item.KeyOffset >= mergeinfo.IndexMergePos)
                                        {
                                            break;
                                        }
                                        if (copylast > 0 && item.KeyOffset >= copylast)
                                        {
                                            break;
                                        }
                                        if (item.KeyOffset == readoldstartpostion)
                                        {
                                            break;
                                        }
                                        if (item.Del)
                                        {
                                            continue;
                                        }
                                        //ProcessTraceUtil.Trace("补中间+1");
                                        lastrankindex++;
                                    }
                                }
                            }
                            else
                            {
                                ProcessTraceUtil.Trace("中间copy2");
                                var offset = 0L;
                                using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
                                {
                                    offset = nw.GetWritePosition() - readoldstartpostion;
                                }
                                IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, readoldstartpostion, copylast - 1, false);

                                foreach (var p in keyindexdisklist[tablename])
                                {
                                    if (p.KeyOffset >= readoldstartpostion && p.KeyOffset < copylast)
                                    {
                                        newdiskindexlist.Add(new BigEntityTableIndexItem
                                        {
                                            Del        = p.Del,
                                            Key        = p.Key,
                                            KeyOffset  = p.KeyOffset + offset,
                                            len        = p.len,
                                            Offset     = p.Offset,
                                            Index      = p.Index,
                                            RangeIndex = p.RangeIndex + readcount
                                        });
                                    }
                                }

                                readoldstartpostion = copylast;
                                ProcessTraceUtil.Trace("中间copy2完成");
                            }
                        }
                    }
                }

                //后面copy
                string tablefile = GetTableFile(tablename);

                var idxreader = ObjTextReader.CreateReader(indexfile);
                var newwriter = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2);
                try
                {
                    long nextcopypos = 0;
                    ProcessTraceUtil.TraceMem("读取后面的数据->" + lasmargepos, "m");
                    idxreader.SetPostion(lasmargepos);
                    bool hasitem   = false;
                    bool isfirst   = true;
                    long keyoffset = 0;
                    foreach (var item in idxreader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, smallbuffer))
                    {
                        hasitem = true;
                        item.SetIndex(meta.KeyIndexInfo);
                        item.KeyOffset = keyoffset;
                        if (item.KeyOffset > newwriter.GetWritePosition())
                        {
                            var spacelen = item.KeyOffset - newwriter.GetWritePosition();
                            ProcessTraceUtil.Trace("没有对齐,尝试对齐,spacelen->" + spacelen);
                            if (spacelen % 3 == 0)
                            {
                                newwriter.FillSpace(spacelen / 3);
                            }
                            else
                            {
                                ProcessTraceUtil.Trace("对齐失败");
                                var ex = new Exception("无法对齐");
                                ex.Data.Add("老索引文件当前位置", item.KeyOffset);
                                ex.Data.Add("新索引文件写入位置", newwriter.GetWritePosition());
                                throw ex;
                            }
                        }

                        if (item.KeyOffset == newwriter.GetWritePosition())
                        {
                            nextcopypos = newwriter.GetWritePosition();
                            if (isfirst)
                            {
                                newIndexMergePos = nextcopypos;
                            }
                            ProcessTraceUtil.Trace("新老索引文件已经对齐:" + item.KeyOffset);
                            break;
                        }

                        isfirst        = false;
                        item.KeyOffset = newwriter.GetWritePosition();
                        newwriter.AppendObject(item);
                    }

                    if (nextcopypos > 0)
                    {
                        newwriter.Dispose();
                        ProcessTraceUtil.Trace("copy后面的数据->" + nextcopypos);
                        nextcopypos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, nextcopypos, -512);
                    }
                    else if (!hasitem)
                    {
                        var idxpos = idxreader.ReadedPostion();
                        if (idxpos == newwriter.GetWritePosition())
                        {
                            nextcopypos = idxpos;
                            newwriter.Dispose();
                        }
                        else
                        {
                            ProcessTraceUtil.Trace(idxpos + " vs " + newwriter.GetWritePosition());
                        }
                    }
                    ProcessTraceUtil.TraceMem("读取后面的数据完成", "m");
                    if (nextcopypos == 0)
                    {
                        throw new Exception("更新索引出错");
                    }

                    try
                    {
                        BigEntityTableIndexItem[] oldindexarray     = null;
                        BigEntityTableIndexItem[] newdiskindexarray = newdiskindexlist.ToArray();

                        tablelocker.EnterWriteLock();
                        ProcessTraceUtil.TraceMem("读取后面的数据->" + lasmargepos, "m");
                        if (nextcopypos <= 0)
                        {
                            using (newwriter)
                            {
                                using (idxreader)
                                {
                                    foreach (var item in idxreader.ReadObjectsWating <BigEntityTableIndexItem>(1, bytes: bigbuffer))
                                    {
                                        item.SetIndex(meta.KeyIndexInfo);
                                        item.KeyOffset = newwriter.GetWritePosition();
                                        newwriter.AppendObject(item);
                                    }
                                }
                            }
                        }
                        else
                        {
                            nextcopypos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, nextcopypos, long.MaxValue);
                            ProcessTraceUtil.TraceMem("继续copy后面的数据->" + nextcopypos, "m");
                            idxreader.Dispose();
                        }
                        ProcessTraceUtil.TraceMem("读取后面的数据完成", "m");

                        //更新索引
                        mergeinfo.LoadFactor = loadFactor;
                        keyindexdisklist.TryRemove(tablename, out oldindexarray);
                        keyindexdisklist.TryAdd(tablename, newdiskindexarray);
                        keyindexmemtemplist[tablename] = new SortArrayList <BigEntityTableIndexItem>();

                        int trycount = 0;
                        while (true)
                        {
                            try
                            {
                                File.Delete(indexfile);
                                ProcessTraceUtil.Trace("删除旧文件完成");
                                break;
                            }
                            catch (System.IO.IOException ex)
                            {
                                Thread.Sleep(1);
                                trycount++;
                                if (trycount > 1000)
                                {
                                    throw ex;
                                }
                            }
                        }

                        File.Move(newindexfile, indexfile);

                        ProcessTraceUtil.TraceMem("删除旧文件,重命名新文件完成", "m");

                        idxreader = null;
                    }
                    finally
                    {
                        tablelocker.ExitWriteLock();
                    }


                    string metafile = GetMetaFile(tablename);
                    mergeinfo.IndexMergePos = lasmargepos;
                    SerializerHelper.SerializerToXML(meta, metafile, true);

                    ProcessTraceUtil.Trace("更新元文件,更新索引完成");
                }
                catch (Exception ex)
                {
                    Console.WriteLine("整理索引后面部分出错" + ex.ToString());

                    //LogManager.LogHelper.Instance.Error("整理索引后面部分出错", ex);
                }
                finally
                {
                    if (idxreader != null)
                    {
                        idxreader.Dispose();
                    }

                    if (newwriter != null && !newwriter.Isdispose)
                    {
                        newwriter.Dispose();
                    }

                    if (File.Exists(newindexfile))
                    {
                        File.Delete(newindexfile);
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());

                //LogManager.LogHelper.Instance.Error("整理索引出错", ex);
            }
            finally
            {
                //GC.Collect();
                //ProcessTraceUtil.TraceMem("回收内存","m");
                mergeinfo.IsMergin = false;
                var info = ProcessTraceUtil.PrintTrace();
                Console.WriteLine(info);
                //LogManager.LogHelper.Instance.Debug("整理索引过程:" + info);
            }
        }
Esempio n. 9
0
        public LocalFileQueue(string queuename, string queuefilepath, bool canwrite = true, bool canread = true)
        {
            if (string.IsNullOrWhiteSpace(queuename))
            {
                throw new ArgumentNullException("queuename");
            }
            QueueName = queuename;

            if (string.IsNullOrWhiteSpace(queuefilepath))
            {
                throw new ArgumentNullException("queuefilepath");
            }
            QueueFilePath = queuefilepath;

            if (!File.Exists(queuefilepath))
            {
                FileInfo file = new FileInfo(queuefilepath);
                if (!file.Directory.Exists)
                {
                    try
                    {
                        file.Directory.Create();
                    }
                    catch (Exception ex)
                    {
                        throw new Exception("创建文件夹失败:" + file.Directory.FullName, ex);
                    }
                }
            }

            if (canwrite)
            {
                //由于文件写是独占式的,当程序快速重启时,文件可能未及时释放
                int trytimes = 0;
                while (true)
                {
                    try
                    {
                        //_queueWriter = ObjTextWriter.CreateWriter(queuefilepath, ObjTextReaderWriterEncodeType.jsonbuf);
                        _queueWriter = ObjTextWriter.CreateWriter(queuefilepath, ObjTextReaderWriterEncodeType.entitybuf);
                        _queueWriter.Flush();
                        break;
                    }
                    catch (Exception ex)
                    {
                        trytimes++;
                        if (trytimes >= 3)
                        {
                            throw ex;
                        }
                        Thread.Sleep(1000 * trytimes);
                    }
                }
            }

            if (canread)
            {
                while (true)
                {
                    if (File.Exists(queuefilepath))
                    {
                        break;
                    }
                    Thread.Sleep(1000);
                }
                _queueReader = ObjTextReader.CreateReader(queuefilepath);

                FileInfo finfo = new FileInfo(queuefilepath);
                QueueCfgFile = finfo.Directory.FullName + "\\" + queuename + ".cfg";
                if (File.Exists(QueueCfgFile))
                {
                    _logger = LJC.FrameWork.Comm.SerializerHelper.DeSerializerFile <LocalFileQueueCfg>(QueueCfgFile, true);
                    if (_logger.LastPos > 0)
                    {
                        _queueReader.SetPostion(_logger.LastPos);
                    }
                }
                else
                {
                    _logger = new LocalFileQueueCfg();
                    _logger.LastChageTime = DateTime.Now;
                    _logger.QueueFile     = queuefilepath;
                    SaveConfig();
                }
            }

            _backtimer = new Timer(new TimerCallback(TimerAction), null, 0, 0);
        }