private void ProcessBadQueue(T last)
{
if (_queueReader == null)
{
return;
}
var oldpostion = _queueReader.ReadedPostion();
while (true)
{
if (_queueReader.PostionNextSplitChar())
{
var newpostion = _queueReader.ReadedPostion();
if (_queueReader.ReadObject <T>() != default(T))
{
_queueReader.SetPostion(newpostion);
OnProcessError(last, new Exception(string.Format("队列损环，将尝试读取下一个队列。损坏位置{0}，恢复位置{1}，损坏长度：{2}kb。", oldpostion, newpostion, (newpostion - oldpostion) / 1000)));
break;
}
}
else
{
throw new Exception("队列已损坏，无法恢复。");
}
}
}
public IEnumerable <T> Find <T>(string tablename, string key) where T : new()
{
string tablefile = GetTableFile(tablename);
EntityTableMeta meta = GetMetaData(tablename);
Dictionary <long, EntityTableIndexItem> arr = null;
EntityTableIndexItem indexitem = null;
if (keyindexdic[tablename].TryGetValue(key, out arr))
{
//先找到offset
using (ObjTextReader otw = ObjTextReader.CreateReader(tablefile))
{
foreach (var o in arr)
{
indexitem = (EntityTableIndexItem)o.Value;
if (!indexitem.Del)
{
otw.SetPostion(indexitem.Offset);
var readobj = otw.ReadObject <EntityTableItem <T> >();
if (readobj == null)
{
yield return(default(T));
}
else
{
yield return(readobj.Data);
}
}
}
}
}
}
private void LoadKey(string tablename, EntityTableMeta meta)
{
string indexfile = GetKeyFile(tablename);
var indexmergeinfo = meta.IndexMergeInfos.Find(p => p.IndexName.Equals(meta.KeyName));
if (indexmergeinfo == null)
{
indexmergeinfo = new IndexMergeInfo();
indexmergeinfo.IndexName = meta.KeyName;
meta.IndexMergeInfos.Add(indexmergeinfo);
}
using (ObjTextReader idx = ObjTextReader.CreateReader(indexfile))
{
if (indexmergeinfo.IndexMergePos > 0)
{
idx.SetPostion(indexmergeinfo.IndexMergePos);
}
var idc = keyindexdic[tablename];
Dictionary <long, EntityTableIndexItem> al = null;
foreach (var newindex in idx.ReadObjectsWating <EntityTableIndexItem>(1))
{
if (!idc.TryGetValue(newindex.Key, out al))
{
lock (idc)
{
if (!idc.TryGetValue(newindex.Key, out al))
{
al = new Dictionary <long, EntityTableIndexItem>();
idc.TryAdd(newindex.Key, al);
}
}
}
if (newindex.Del)
{
al.Remove(newindex.Offset);
}
else
{
lock (al)
{
al.Add(newindex.Offset, newindex);
}
}
}
}
}
private EntityTableIndexItemBag LoadIndex(string tablename, string indexname, EntityTableMeta meta)
{
string key = string.Format("{0}##{1}", tablename, indexname);
EntityTableIndexItemBag temp = null;
if (indexdic.TryGetValue(key, out temp))
{
temp.LastUsed = DateTime.Now;
}
string indexfile = GetIndexFile(tablename, indexname);
var locker = GetKeyLocker(tablename, "index_" + indexname);
lock (locker)
{
if (indexdic.TryGetValue(key, out temp))
{
temp.LastUsed = DateTime.Now;
}
else
{
temp = new EntityTableIndexItemBag();
}
using (ObjTextReader idxreader = ObjTextReader.CreateReader(indexfile))
{
if (temp.LastOffset > 0)
{
idxreader.SetPostion(temp.LastOffset);
}
Dictionary <long, EntityTableIndexItem> al = null;
foreach (var newindex in idxreader.ReadObjectsWating <EntityTableIndexItem>(1))
{
temp.LastOffset = idxreader.ReadedPostion();
if (!temp.Dics.TryGetValue(newindex.Key, out al))
{
al = new Dictionary <long, EntityTableIndexItem>();
temp.Dics.TryAdd(newindex.Key, al);
}
if (newindex.Del)
{
al.Remove(newindex.Offset);
}
else
{
al.Add(newindex.Offset, newindex);
}
}
}
if (temp.LastUsed == DateTime.MinValue)
{
temp.LastUsed = DateTime.Now;
indexdic.TryAdd(key, temp);
LJC.FrameWork.Comm.Coroutine.CoroutineEngine.DefaultCoroutineEngine.Dispatcher(new IndexDestroy(indexdic, key));
}
else
{
temp.LastUsed = DateTime.Now;
}
}
return(temp);
}
private void LoadKey(string tablename, BigEntityTableMeta meta)
{
string indexfile = GetKeyFile(tablename);
var indexmergeinfo = meta.IndexMergeInfos.Find(p => p.IndexName.Equals(meta.KeyName));
if (indexmergeinfo == null)
{
indexmergeinfo = new IndexMergeInfo();
indexmergeinfo.IndexName = meta.KeyName;
meta.IndexMergeInfos.Add(indexmergeinfo);
}
//计算加载因子
indexmergeinfo.LoadFactor = (int)Math.Max(4, new FileInfo(indexfile).Length / MAX_KEYBUFFER);
int i = 0;
BigEntityTableIndexItem lastreadindex = null;
List <BigEntityTableIndexItem> list = new List <BigEntityTableIndexItem>();
long currentpos = 0;
long currrankindex = 0;
byte[] buffer = new byte[1024 * 1024 * 10];
using (ObjTextReader idx = ObjTextReader.CreateReader(indexfile))
{
Console.WriteLine("loadkey");
foreach (var newindex in idx.ReadObjectsWating <BigEntityTableIndexItem>(1, p => currentpos = p, buffer))
{
if (newindex.Del)
{
continue;
}
newindex.KeyOffset = currentpos;
newindex.RangeIndex = currrankindex++;
newindex.SetIndex(meta.KeyIndexInfo);
if (newindex.KeyOffset >= indexmergeinfo.IndexMergePos)
{
//list.Add(newindex);
if (list.Count > 0)
{
if (list.Last().KeyOffset != lastreadindex.KeyOffset)
{
list.Add(lastreadindex);
}
}
break;
}
if (indexmergeinfo.LoadFactor == 1 || i % indexmergeinfo.LoadFactor == 0)
{
list.Add(newindex);
}
i++;
lastreadindex = newindex;
}
if (list.Count > 0 && list.Last().KeyOffset != lastreadindex.KeyOffset)
{
list.Add(lastreadindex);
}
}
indexmergeinfo.TotalCount = i;
BigEntityTableIndexItem[] oldindexitems = null;
keyindexdisklist.TryRemove(tablename, out oldindexitems);
keyindexdisklist.TryAdd(tablename, list.ToArray());
using (ObjTextReader idr = ObjTextReader.CreateReader(indexfile))
{
Console.WriteLine("loadkey2");
if (indexmergeinfo.IndexMergePos > 0)
{
idr.SetPostion(indexmergeinfo.IndexMergePos);
}
//Dictionary<string, BigEntityTableIndexItem> indexdic = new Dictionary<string, BigEntityTableIndexItem>();
//keyindexlistdic[tablename];
SortArrayList <BigEntityTableIndexItem> keymemlist = new SortArrayList <BigEntityTableIndexItem>();
foreach (var newindex in idr.ReadObjectsWating <BigEntityTableIndexItem>(1, p => currentpos = p, buffer))
{
newindex.KeyOffset = currentpos;
newindex.SetIndex(meta.KeyIndexInfo);
newindex.RangeIndex = -1;
if (!newindex.Del)
{
//indexdic.Add(newindex.Key, newindex);
keymemlist.Add(newindex);
i++;
}
}
var tablelocker = GetKeyLocker(tablename, string.Empty);
tablelocker.EnterWriteLock();
try
{
foreach (var newindex in idr.ReadObjectsWating <BigEntityTableIndexItem>(1, p => currentpos = p, buffer))
{
newindex.KeyOffset = currentpos;
newindex.SetIndex(meta.KeyIndexInfo);
newindex.RangeIndex = -1;
if (!newindex.Del)
{
//indexdic.Add(newindex.Key, newindex);
keymemlist.Add(newindex);
i++;
}
}
if (idr.Length() - currentpos > 10240)
{
throw new Exception(tablename + "主键索引大量数据未读取");
}
//keyindexlistdic[tablename] = indexdic;
keyindexmemlist[tablename] = keymemlist;
}
finally
{
tablelocker.ExitWriteLock();
}
}
}
public LocalFileQueue(string queuename, string queuefilepath, bool canwrite = true, bool canread = true)
{
if (string.IsNullOrWhiteSpace(queuename))
{
throw new ArgumentNullException("queuename");
}
QueueName = queuename;
if (string.IsNullOrWhiteSpace(queuefilepath))
{
throw new ArgumentNullException("queuefilepath");
}
QueueFilePath = queuefilepath;
if (!File.Exists(queuefilepath))
{
FileInfo file = new FileInfo(queuefilepath);
if (!file.Directory.Exists)
{
try
{
file.Directory.Create();
}
catch (Exception ex)
{
throw new Exception("创建文件夹失败:" + file.Directory.FullName, ex);
}
}
}
if (canwrite)
{
//由于文件写是独占式的，当程序快速重启时，文件可能未及时释放
int trytimes = 0;
while (true)
{
try
{
//_queueWriter = ObjTextWriter.CreateWriter(queuefilepath, ObjTextReaderWriterEncodeType.jsonbuf);
_queueWriter = ObjTextWriter.CreateWriter(queuefilepath, ObjTextReaderWriterEncodeType.entitybuf);
_queueWriter.Flush();
break;
}
catch (Exception ex)
{
trytimes++;
if (trytimes >= 3)
{
throw ex;
}
Thread.Sleep(1000 * trytimes);
}
}
}
if (canread)
{
while (true)
{
if (File.Exists(queuefilepath))
{
break;
}
Thread.Sleep(1000);
}
_queueReader = ObjTextReader.CreateReader(queuefilepath);
FileInfo finfo = new FileInfo(queuefilepath);
QueueCfgFile = finfo.Directory.FullName + "\\" + queuename + ".cfg";
if (File.Exists(QueueCfgFile))
{
_logger = LJC.FrameWork.Comm.SerializerHelper.DeSerializerFile <LocalFileQueueCfg>(QueueCfgFile, true);
if (_logger.LastPos > 0)
{
_queueReader.SetPostion(_logger.LastPos);
}
}
else
{
_logger = new LocalFileQueueCfg();
_logger.LastChageTime = DateTime.Now;
_logger.QueueFile = queuefilepath;
SaveConfig();
}
}
_backtimer = new Timer(new TimerCallback(TimerAction), null, 0, 0);
}