private void ProcessBadQueue(T last) { if (_queueReader == null) { return; } var oldpostion = _queueReader.ReadedPostion(); while (true) { if (_queueReader.PostionNextSplitChar()) { var newpostion = _queueReader.ReadedPostion(); if (_queueReader.ReadObject <T>() != default(T)) { _queueReader.SetPostion(newpostion); OnProcessError(last, new Exception(string.Format("队列损环,将尝试读取下一个队列。损坏位置{0},恢复位置{1},损坏长度:{2}kb。", oldpostion, newpostion, (newpostion - oldpostion) / 1000))); break; } } else { throw new Exception("队列已损坏,无法恢复。"); } } }
public IEnumerable <T> Find <T>(string tablename, string key) where T : new() { string tablefile = GetTableFile(tablename); EntityTableMeta meta = GetMetaData(tablename); Dictionary <long, EntityTableIndexItem> arr = null; EntityTableIndexItem indexitem = null; if (keyindexdic[tablename].TryGetValue(key, out arr)) { //先找到offset using (ObjTextReader otw = ObjTextReader.CreateReader(tablefile)) { foreach (var o in arr) { indexitem = (EntityTableIndexItem)o.Value; if (!indexitem.Del) { otw.SetPostion(indexitem.Offset); var readobj = otw.ReadObject <EntityTableItem <T> >(); if (readobj == null) { yield return(default(T)); } else { yield return(readobj.Data); } } } } } }
private void LoadKey(string tablename, EntityTableMeta meta) { string indexfile = GetKeyFile(tablename); var indexmergeinfo = meta.IndexMergeInfos.Find(p => p.IndexName.Equals(meta.KeyName)); if (indexmergeinfo == null) { indexmergeinfo = new IndexMergeInfo(); indexmergeinfo.IndexName = meta.KeyName; meta.IndexMergeInfos.Add(indexmergeinfo); } using (ObjTextReader idx = ObjTextReader.CreateReader(indexfile)) { if (indexmergeinfo.IndexMergePos > 0) { idx.SetPostion(indexmergeinfo.IndexMergePos); } var idc = keyindexdic[tablename]; Dictionary <long, EntityTableIndexItem> al = null; foreach (var newindex in idx.ReadObjectsWating <EntityTableIndexItem>(1)) { if (!idc.TryGetValue(newindex.Key, out al)) { lock (idc) { if (!idc.TryGetValue(newindex.Key, out al)) { al = new Dictionary <long, EntityTableIndexItem>(); idc.TryAdd(newindex.Key, al); } } } if (newindex.Del) { al.Remove(newindex.Offset); } else { lock (al) { al.Add(newindex.Offset, newindex); } } } } }
private EntityTableIndexItemBag LoadIndex(string tablename, string indexname, EntityTableMeta meta) { string key = string.Format("{0}##{1}", tablename, indexname); EntityTableIndexItemBag temp = null; if (indexdic.TryGetValue(key, out temp)) { temp.LastUsed = DateTime.Now; } string indexfile = GetIndexFile(tablename, indexname); var locker = GetKeyLocker(tablename, "index_" + indexname); lock (locker) { if (indexdic.TryGetValue(key, out temp)) { temp.LastUsed = DateTime.Now; } else { temp = new EntityTableIndexItemBag(); } using (ObjTextReader idxreader = ObjTextReader.CreateReader(indexfile)) { if (temp.LastOffset > 0) { idxreader.SetPostion(temp.LastOffset); } Dictionary <long, EntityTableIndexItem> al = null; foreach (var newindex in idxreader.ReadObjectsWating <EntityTableIndexItem>(1)) { temp.LastOffset = idxreader.ReadedPostion(); if (!temp.Dics.TryGetValue(newindex.Key, out al)) { al = new Dictionary <long, EntityTableIndexItem>(); temp.Dics.TryAdd(newindex.Key, al); } if (newindex.Del) { al.Remove(newindex.Offset); } else { al.Add(newindex.Offset, newindex); } } } if (temp.LastUsed == DateTime.MinValue) { temp.LastUsed = DateTime.Now; indexdic.TryAdd(key, temp); LJC.FrameWork.Comm.Coroutine.CoroutineEngine.DefaultCoroutineEngine.Dispatcher(new IndexDestroy(indexdic, key)); } else { temp.LastUsed = DateTime.Now; } } return(temp); }
private void LoadKey(string tablename, BigEntityTableMeta meta) { string indexfile = GetKeyFile(tablename); var indexmergeinfo = meta.IndexMergeInfos.Find(p => p.IndexName.Equals(meta.KeyName)); if (indexmergeinfo == null) { indexmergeinfo = new IndexMergeInfo(); indexmergeinfo.IndexName = meta.KeyName; meta.IndexMergeInfos.Add(indexmergeinfo); } //计算加载因子 indexmergeinfo.LoadFactor = (int)Math.Max(4, new FileInfo(indexfile).Length / MAX_KEYBUFFER); int i = 0; BigEntityTableIndexItem lastreadindex = null; List <BigEntityTableIndexItem> list = new List <BigEntityTableIndexItem>(); long currentpos = 0; long currrankindex = 0; byte[] buffer = new byte[1024 * 1024 * 10]; using (ObjTextReader idx = ObjTextReader.CreateReader(indexfile)) { Console.WriteLine("loadkey"); foreach (var newindex in idx.ReadObjectsWating <BigEntityTableIndexItem>(1, p => currentpos = p, buffer)) { if (newindex.Del) { continue; } newindex.KeyOffset = currentpos; newindex.RangeIndex = currrankindex++; newindex.SetIndex(meta.KeyIndexInfo); if (newindex.KeyOffset >= indexmergeinfo.IndexMergePos) { //list.Add(newindex); if (list.Count > 0) { if (list.Last().KeyOffset != lastreadindex.KeyOffset) { list.Add(lastreadindex); } } break; } if (indexmergeinfo.LoadFactor == 1 || i % indexmergeinfo.LoadFactor == 0) { list.Add(newindex); } i++; lastreadindex = newindex; } if (list.Count > 0 && list.Last().KeyOffset != lastreadindex.KeyOffset) { list.Add(lastreadindex); } } indexmergeinfo.TotalCount = i; BigEntityTableIndexItem[] oldindexitems = null; keyindexdisklist.TryRemove(tablename, out oldindexitems); keyindexdisklist.TryAdd(tablename, list.ToArray()); using (ObjTextReader idr = ObjTextReader.CreateReader(indexfile)) { Console.WriteLine("loadkey2"); if (indexmergeinfo.IndexMergePos > 0) { idr.SetPostion(indexmergeinfo.IndexMergePos); } //Dictionary<string, BigEntityTableIndexItem> indexdic = new Dictionary<string, BigEntityTableIndexItem>(); //keyindexlistdic[tablename]; SortArrayList <BigEntityTableIndexItem> keymemlist = new SortArrayList <BigEntityTableIndexItem>(); foreach (var newindex in idr.ReadObjectsWating <BigEntityTableIndexItem>(1, p => currentpos = p, buffer)) { newindex.KeyOffset = currentpos; newindex.SetIndex(meta.KeyIndexInfo); newindex.RangeIndex = -1; if (!newindex.Del) { //indexdic.Add(newindex.Key, newindex); keymemlist.Add(newindex); i++; } } var tablelocker = GetKeyLocker(tablename, string.Empty); tablelocker.EnterWriteLock(); try { foreach (var newindex in idr.ReadObjectsWating <BigEntityTableIndexItem>(1, p => currentpos = p, buffer)) { newindex.KeyOffset = currentpos; newindex.SetIndex(meta.KeyIndexInfo); newindex.RangeIndex = -1; if (!newindex.Del) { //indexdic.Add(newindex.Key, newindex); keymemlist.Add(newindex); i++; } } if (idr.Length() - currentpos > 10240) { throw new Exception(tablename + "主键索引大量数据未读取"); } //keyindexlistdic[tablename] = indexdic; keyindexmemlist[tablename] = keymemlist; } finally { tablelocker.ExitWriteLock(); } } }
public LocalFileQueue(string queuename, string queuefilepath, bool canwrite = true, bool canread = true) { if (string.IsNullOrWhiteSpace(queuename)) { throw new ArgumentNullException("queuename"); } QueueName = queuename; if (string.IsNullOrWhiteSpace(queuefilepath)) { throw new ArgumentNullException("queuefilepath"); } QueueFilePath = queuefilepath; if (!File.Exists(queuefilepath)) { FileInfo file = new FileInfo(queuefilepath); if (!file.Directory.Exists) { try { file.Directory.Create(); } catch (Exception ex) { throw new Exception("创建文件夹失败:" + file.Directory.FullName, ex); } } } if (canwrite) { //由于文件写是独占式的,当程序快速重启时,文件可能未及时释放 int trytimes = 0; while (true) { try { //_queueWriter = ObjTextWriter.CreateWriter(queuefilepath, ObjTextReaderWriterEncodeType.jsonbuf); _queueWriter = ObjTextWriter.CreateWriter(queuefilepath, ObjTextReaderWriterEncodeType.entitybuf); _queueWriter.Flush(); break; } catch (Exception ex) { trytimes++; if (trytimes >= 3) { throw ex; } Thread.Sleep(1000 * trytimes); } } } if (canread) { while (true) { if (File.Exists(queuefilepath)) { break; } Thread.Sleep(1000); } _queueReader = ObjTextReader.CreateReader(queuefilepath); FileInfo finfo = new FileInfo(queuefilepath); QueueCfgFile = finfo.Directory.FullName + "\\" + queuename + ".cfg"; if (File.Exists(QueueCfgFile)) { _logger = LJC.FrameWork.Comm.SerializerHelper.DeSerializerFile <LocalFileQueueCfg>(QueueCfgFile, true); if (_logger.LastPos > 0) { _queueReader.SetPostion(_logger.LastPos); } } else { _logger = new LocalFileQueueCfg(); _logger.LastChageTime = DateTime.Now; _logger.QueueFile = queuefilepath; SaveConfig(); } } _backtimer = new Timer(new TimerCallback(TimerAction), null, 0, 0); }