private void LoadKey(string tablename, EntityTableMeta meta)
{
string indexfile = GetKeyFile(tablename);
var indexmergeinfo = meta.IndexMergeInfos.Find(p => p.IndexName.Equals(meta.KeyName));
if (indexmergeinfo == null)
{
indexmergeinfo = new IndexMergeInfo();
indexmergeinfo.IndexName = meta.KeyName;
meta.IndexMergeInfos.Add(indexmergeinfo);
}
using (ObjTextReader idx = ObjTextReader.CreateReader(indexfile))
{
if (indexmergeinfo.IndexMergePos > 0)
{
idx.SetPostion(indexmergeinfo.IndexMergePos);
}
var idc = keyindexdic[tablename];
Dictionary <long, EntityTableIndexItem> al = null;
foreach (var newindex in idx.ReadObjectsWating <EntityTableIndexItem>(1))
{
if (!idc.TryGetValue(newindex.Key, out al))
{
lock (idc)
{
if (!idc.TryGetValue(newindex.Key, out al))
{
al = new Dictionary <long, EntityTableIndexItem>();
idc.TryAdd(newindex.Key, al);
}
}
}
if (newindex.Del)
{
al.Remove(newindex.Offset);
}
else
{
lock (al)
{
al.Add(newindex.Offset, newindex);
}
}
}
}
}
private EntityTableIndexItemBag LoadIndex(string tablename, string indexname, EntityTableMeta meta)
{
string key = string.Format("{0}##{1}", tablename, indexname);
EntityTableIndexItemBag temp = null;
if (indexdic.TryGetValue(key, out temp))
{
temp.LastUsed = DateTime.Now;
}
string indexfile = GetIndexFile(tablename, indexname);
var locker = GetKeyLocker(tablename, "index_" + indexname);
lock (locker)
{
if (indexdic.TryGetValue(key, out temp))
{
temp.LastUsed = DateTime.Now;
}
else
{
temp = new EntityTableIndexItemBag();
}
using (ObjTextReader idxreader = ObjTextReader.CreateReader(indexfile))
{
if (temp.LastOffset > 0)
{
idxreader.SetPostion(temp.LastOffset);
}
Dictionary <long, EntityTableIndexItem> al = null;
foreach (var newindex in idxreader.ReadObjectsWating <EntityTableIndexItem>(1))
{
temp.LastOffset = idxreader.ReadedPostion();
if (!temp.Dics.TryGetValue(newindex.Key, out al))
{
al = new Dictionary <long, EntityTableIndexItem>();
temp.Dics.TryAdd(newindex.Key, al);
}
if (newindex.Del)
{
al.Remove(newindex.Offset);
}
else
{
al.Add(newindex.Offset, newindex);
}
}
}
if (temp.LastUsed == DateTime.MinValue)
{
temp.LastUsed = DateTime.Now;
indexdic.TryAdd(key, temp);
LJC.FrameWork.Comm.Coroutine.CoroutineEngine.DefaultCoroutineEngine.Dispatcher(new IndexDestroy(indexdic, key));
}
else
{
temp.LastUsed = DateTime.Now;
}
}
return(temp);
}
private void LoadKey(string tablename, BigEntityTableMeta meta)
{
string indexfile = GetKeyFile(tablename);
var indexmergeinfo = meta.IndexMergeInfos.Find(p => p.IndexName.Equals(meta.KeyName));
if (indexmergeinfo == null)
{
indexmergeinfo = new IndexMergeInfo();
indexmergeinfo.IndexName = meta.KeyName;
meta.IndexMergeInfos.Add(indexmergeinfo);
}
//计算加载因子
indexmergeinfo.LoadFactor = (int)Math.Max(4, new FileInfo(indexfile).Length / MAX_KEYBUFFER);
int i = 0;
BigEntityTableIndexItem lastreadindex = null;
List <BigEntityTableIndexItem> list = new List <BigEntityTableIndexItem>();
long currentpos = 0;
long currrankindex = 0;
byte[] buffer = new byte[1024 * 1024 * 10];
using (ObjTextReader idx = ObjTextReader.CreateReader(indexfile))
{
Console.WriteLine("loadkey");
foreach (var newindex in idx.ReadObjectsWating <BigEntityTableIndexItem>(1, p => currentpos = p, buffer))
{
if (newindex.Del)
{
continue;
}
newindex.KeyOffset = currentpos;
newindex.RangeIndex = currrankindex++;
newindex.SetIndex(meta.KeyIndexInfo);
if (newindex.KeyOffset >= indexmergeinfo.IndexMergePos)
{
//list.Add(newindex);
if (list.Count > 0)
{
if (list.Last().KeyOffset != lastreadindex.KeyOffset)
{
list.Add(lastreadindex);
}
}
break;
}
if (indexmergeinfo.LoadFactor == 1 || i % indexmergeinfo.LoadFactor == 0)
{
list.Add(newindex);
}
i++;
lastreadindex = newindex;
}
if (list.Count > 0 && list.Last().KeyOffset != lastreadindex.KeyOffset)
{
list.Add(lastreadindex);
}
}
indexmergeinfo.TotalCount = i;
BigEntityTableIndexItem[] oldindexitems = null;
keyindexdisklist.TryRemove(tablename, out oldindexitems);
keyindexdisklist.TryAdd(tablename, list.ToArray());
using (ObjTextReader idr = ObjTextReader.CreateReader(indexfile))
{
Console.WriteLine("loadkey2");
if (indexmergeinfo.IndexMergePos > 0)
{
idr.SetPostion(indexmergeinfo.IndexMergePos);
}
//Dictionary<string, BigEntityTableIndexItem> indexdic = new Dictionary<string, BigEntityTableIndexItem>();
//keyindexlistdic[tablename];
SortArrayList <BigEntityTableIndexItem> keymemlist = new SortArrayList <BigEntityTableIndexItem>();
foreach (var newindex in idr.ReadObjectsWating <BigEntityTableIndexItem>(1, p => currentpos = p, buffer))
{
newindex.KeyOffset = currentpos;
newindex.SetIndex(meta.KeyIndexInfo);
newindex.RangeIndex = -1;
if (!newindex.Del)
{
//indexdic.Add(newindex.Key, newindex);
keymemlist.Add(newindex);
i++;
}
}
var tablelocker = GetKeyLocker(tablename, string.Empty);
tablelocker.EnterWriteLock();
try
{
foreach (var newindex in idr.ReadObjectsWating <BigEntityTableIndexItem>(1, p => currentpos = p, buffer))
{
newindex.KeyOffset = currentpos;
newindex.SetIndex(meta.KeyIndexInfo);
newindex.RangeIndex = -1;
if (!newindex.Del)
{
//indexdic.Add(newindex.Key, newindex);
keymemlist.Add(newindex);
i++;
}
}
if (idr.Length() - currentpos > 10240)
{
throw new Exception(tablename + "主键索引大量数据未读取");
}
//keyindexlistdic[tablename] = indexdic;
keyindexmemlist[tablename] = keymemlist;
}
finally
{
tablelocker.ExitWriteLock();
}
}
}
private void ProcessQueue()
{
if (OnProcessQueue == null)
{
return;
}
lock (this)
{
if (IsRuning)
{
return;
}
IsRuning = true;
}
T last = null;
int errortimes = 0;
try
{
//ProcessBadQueue(last);
foreach (var t in _queueReader.ReadObjectsWating <T>())
{
last = t;
if (OnProcessQueue(t))
{
errortimes = 0;
_logger.LastPos = _queueReader.ReadedPostion();
_logger.LastChageTime = DateTime.Now;
if (OnProcessQueueSuccessed != null)
{
OnProcessQueueSuccessed(t);
}
}
else
{
_queueReader.SetPostion(_logger.LastPos);
errortimes++;
}
if (errortimes > 0)
{
Thread.Sleep(1000 * errortimes);
}
if (errortimes > 5)
{
throw new Exception("尝试次数过多");
}
}
}
catch (Newtonsoft.Json.JsonReaderException)
{
ProcessBadQueue(last);
}
catch (Newtonsoft.Json.JsonSerializationException)
{
ProcessBadQueue(last);
}
catch (EntityBuf.EntityBufException)
{
ProcessBadQueue(last);
}
catch (Exception ex)
{
if (!OnErrorResumeNext)
{
_queueReader.SetPostion(_logger.LastPos);
}
if (OnProcessError != null)
{
OnProcessError(last, ex);
}
}
finally
{
IsRuning = false;
}
}
