public IEnumerable <T> Find <T>(string tablename, string key) where T : new()
{
string tablefile = GetTableFile(tablename);
EntityTableMeta meta = GetMetaData(tablename);
Dictionary <long, EntityTableIndexItem> arr = null;
EntityTableIndexItem indexitem = null;
if (keyindexdic[tablename].TryGetValue(key, out arr))
{
//先找到offset
using (ObjTextReader otw = ObjTextReader.CreateReader(tablefile))
{
foreach (var o in arr)
{
indexitem = (EntityTableIndexItem)o.Value;
if (!indexitem.Del)
{
otw.SetPostion(indexitem.Offset);
var readobj = otw.ReadObject <EntityTableItem <T> >();
if (readobj == null)
{
yield return(default(T));
}
else
{
yield return(readobj.Data);
}
}
}
}
}
}
private void LoadKey(string tablename, EntityTableMeta meta)
{
string indexfile = GetKeyFile(tablename);
var indexmergeinfo = meta.IndexMergeInfos.Find(p => p.IndexName.Equals(meta.KeyName));
if (indexmergeinfo == null)
{
indexmergeinfo = new IndexMergeInfo();
indexmergeinfo.IndexName = meta.KeyName;
meta.IndexMergeInfos.Add(indexmergeinfo);
}
using (ObjTextReader idx = ObjTextReader.CreateReader(indexfile))
{
if (indexmergeinfo.IndexMergePos > 0)
{
idx.SetPostion(indexmergeinfo.IndexMergePos);
}
var idc = keyindexdic[tablename];
Dictionary <long, EntityTableIndexItem> al = null;
foreach (var newindex in idx.ReadObjectsWating <EntityTableIndexItem>(1))
{
if (!idc.TryGetValue(newindex.Key, out al))
{
lock (idc)
{
if (!idc.TryGetValue(newindex.Key, out al))
{
al = new Dictionary <long, EntityTableIndexItem>();
idc.TryAdd(newindex.Key, al);
}
}
}
if (newindex.Del)
{
al.Remove(newindex.Offset);
}
else
{
lock (al)
{
al.Add(newindex.Offset, newindex);
}
}
}
}
}
private EntityTableIndexItemBag LoadIndex(string tablename, string indexname, EntityTableMeta meta)
{
string key = string.Format("{0}##{1}", tablename, indexname);
EntityTableIndexItemBag temp = null;
if (indexdic.TryGetValue(key, out temp))
{
temp.LastUsed = DateTime.Now;
}
string indexfile = GetIndexFile(tablename, indexname);
var locker = GetKeyLocker(tablename, "index_" + indexname);
lock (locker)
{
if (indexdic.TryGetValue(key, out temp))
{
temp.LastUsed = DateTime.Now;
}
else
{
temp = new EntityTableIndexItemBag();
}
using (ObjTextReader idxreader = ObjTextReader.CreateReader(indexfile))
{
if (temp.LastOffset > 0)
{
idxreader.SetPostion(temp.LastOffset);
}
Dictionary <long, EntityTableIndexItem> al = null;
foreach (var newindex in idxreader.ReadObjectsWating <EntityTableIndexItem>(1))
{
temp.LastOffset = idxreader.ReadedPostion();
if (!temp.Dics.TryGetValue(newindex.Key, out al))
{
al = new Dictionary <long, EntityTableIndexItem>();
temp.Dics.TryAdd(newindex.Key, al);
}
if (newindex.Del)
{
al.Remove(newindex.Offset);
}
else
{
al.Add(newindex.Offset, newindex);
}
}
}
if (temp.LastUsed == DateTime.MinValue)
{
temp.LastUsed = DateTime.Now;
indexdic.TryAdd(key, temp);
LJC.FrameWork.Comm.Coroutine.CoroutineEngine.DefaultCoroutineEngine.Dispatcher(new IndexDestroy(indexdic, key));
}
else
{
temp.LastUsed = DateTime.Now;
}
}
return(temp);
}
public void MergeKey2(string tablename, string indexname, BigEntityTableMeta meta)
{
ProcessTraceUtil.StartTrace();
IndexMergeInfo mergeinfo = null;
lock (meta)
{
mergeinfo = meta.IndexMergeInfos.Find(p => indexname.Equals(p.IndexName));
if (mergeinfo == null)
{
mergeinfo = new IndexMergeInfo();
mergeinfo.IndexName = indexname;
meta.IndexMergeInfos.Add(mergeinfo);
}
if (mergeinfo.IsMergin)
{
return;
}
meta.NewAddCount = 0;
mergeinfo.IsMergin = true;
}
DateTime timestart = DateTime.Now;
string newindexfile = string.Empty;
byte[] bigbuffer = new byte[1024 * 1024];
byte[] smallbuffer = new byte[2048];
try
{
ProcessTraceUtil.TraceMem("开始整理key:" + indexname, "m");
long lasmargepos = 0;
long newIndexMergePos = mergeinfo.IndexMergePos;
string indexfile = indexname.Equals(meta.KeyName) ? GetKeyFile(tablename) : GetIndexFile(tablename, indexname);
var tablelocker = GetKeyLocker(tablename, string.Empty);
//新的磁盘索引
List <BigEntityTableIndexItem> newdiskindexlist = new List <BigEntityTableIndexItem>();
var loadFactor = (int)Math.Max(4, new FileInfo(indexfile).Length / MAX_KEYBUFFER);
using (var reader = ObjTextReader.CreateReader(indexfile))
{
long readoldstartpostion = reader.ReadedPostion();
try
{
tablelocker.EnterWriteLock();
keyindexmemtemplist[tablename] = keyindexmemlist[tablename];
keyindexmemlist[tablename] = new SortArrayList <BigEntityTableIndexItem>();
}
finally
{
tablelocker.ExitWriteLock();
}
var listtemp = keyindexmemtemplist[tablename].GetList().ToList();
listtemp = listtemp.Select(p => new BigEntityTableIndexItem
{
Del = p.Del,
Key = p.Key,
KeyOffset = p.KeyOffset,
len = p.len,
Offset = p.Offset,
Index = p.Index,
RangeIndex = p.RangeIndex
}).ToList();
int readcount = listtemp.Count;
ProcessTraceUtil.TraceMem("从内存中读取新增数据，共" + readcount + "条", "m");
if (readcount == 0)
{
return;
}
lasmargepos = listtemp.Max(p => p.KeyOffset);
reader.SetPostion(lasmargepos);
reader.ReadObject <BigEntityTableIndexItem>();
lasmargepos = reader.ReadedPostion();
//优化确定哪些部分是不需要一个个读入的
long copybefore = 0, copylast = 0;
long lastrankindex = 0;
var indexarray = keyindexdisklist[tablename];
using (var sortarray = new Collections.SorteArray <BigEntityTableIndexItem>(indexarray))
{
int mid = -1;
int pos = sortarray.Find(listtemp.First(), ref mid);
ProcessTraceUtil.Trace("查找新数据在老数中插入的开始位置:mid=" + mid + ",pos=" + pos);
if (pos == -1 && mid != -1)
{
//小于最小的
copybefore = indexarray[mid].KeyOffset;
lastrankindex = indexarray[mid].RangeIndex;
ProcessTraceUtil.Trace("老数据可以直接copy的部分:0->" + copybefore);
}
else if (pos != -1)
{
copybefore = indexarray[pos].KeyOffset;
lastrankindex = indexarray[pos].RangeIndex;
ProcessTraceUtil.Trace("老数据可以直接copy的部分:0->" + copybefore);
}
//优化确定后面读到哪
mid = -1;
pos = sortarray.Find(listtemp.Last(), ref mid);
ProcessTraceUtil.Trace("查找新数据在老数据中插入的结束位置:mid=" + mid + ",pos=" + pos);
if (pos == -1 && mid != -1 && mid < indexarray.Length - 1)
{
//小于最小的
copylast = indexarray[mid + 1].KeyOffset;
ProcessTraceUtil.Trace("老数据可以直接copy的部分:" + copylast + "->" + mergeinfo.IndexMergePos);
}
else if (pos != -1)
{
copylast = indexarray[pos].KeyOffset;
ProcessTraceUtil.Trace("老数据可以直接copy的部分:" + copylast + "->" + mergeinfo.IndexMergePos);
}
}
newindexfile = (indexname.Equals(meta.KeyName) ? GetKeyFile(tablename) : GetIndexFile(tablename, indexname)) + ".temp";
if (File.Exists(newindexfile))
{
File.Delete(newindexfile);
}
//快速copy
if (copybefore > 0)
{
ProcessTraceUtil.TraceMem("直接copy前面不在排序范围的数据:0->" + copybefore, "m");
IOUtil.CopyFile(indexfile, newindexfile, FileMode.Create, 0, copybefore - 1);
readoldstartpostion = copybefore;
ProcessTraceUtil.TraceMem("copy数据完成", "m");
newdiskindexlist.AddRange(keyindexdisklist[tablename].Where(p => p.KeyOffset < copybefore));
}
bool isall = false;
ProcessTraceUtil.TraceMem("开始读取在排序范围内的数据", "m");
while (true)
{
ProcessTraceUtil.Trace("读取老数据,开始位置:" + readoldstartpostion);
reader.SetPostion(readoldstartpostion);
var listordered = new List <BigEntityTableIndexItem>();
var loadcount = 0;
long keyoffset = 0;
foreach (var item in reader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, bigbuffer))
{
item.KeyOffset = keyoffset;
item.SetIndex(meta.KeyIndexInfo);
if (item.KeyOffset >= mergeinfo.IndexMergePos)
{
break;
}
if (copylast > 0 && item.KeyOffset >= copylast)
{
break;
}
listordered.Add(item);
if (++loadcount >= MERGE_TRIGGER_NEW_COUNT)
{
break;
}
}
readoldstartpostion = reader.ReadedPostion();
bool isonlyoldlist = false;
if (listordered.Count == 0)
{
ProcessTraceUtil.TraceMem("老数据没有了，全部是新数据:" + listtemp.Count, "m");
listordered = MergeAndSort2(listordered, listtemp).ToList();
foreach (var item in listordered)
{
item.RangeIndex = lastrankindex++;
}
isall = true;
}
else if (listtemp.Count == 0 && listordered.Count > 10000)
{
ProcessTraceUtil.TraceMem("新数据没有了，全部是老数据:" + listordered.Count, "m");
//copy
IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, listordered.First().KeyOffset, listordered.Last().KeyOffset - 1, false);
var listorderedlast = listordered.Last();
long copyoffset = 0;
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
var item = new BigEntityTableIndexItem {
Del = listorderedlast.Del, Key = listorderedlast.Key, len = listorderedlast.len, Offset = listorderedlast.Offset, Index = listorderedlast.Index
};
item.KeyOffset = nw.GetWritePosition();
copyoffset = nw.GetWritePosition() - listorderedlast.KeyOffset;
nw.AppendObject(item);
newIndexMergePos = nw.GetWritePosition();
}
isonlyoldlist = true;
//更新索引
foreach (var item in listordered)
{
item.RangeIndex = lastrankindex++;
item.KeyOffset += copyoffset;
}
ProcessTraceUtil.TraceMem("直接copy数据完成", "m");
}
else
{
ProcessTraceUtil.TraceMem("老数据条数为:" + listordered.Count + "，新数据条数为:" + listtemp.Count, "m");
var listordermax = listordered.Last();
int idx = 0;
foreach (var item in listtemp)
{
if (item.CompareTo(listordermax) > 0)
{
break;
}
else
{
idx++;
}
}
List <BigEntityTableIndexItem> smalllist = listtemp.Take(idx).ToList();
listtemp = listtemp.Skip(idx).ToList();
listordered = MergeAndSort2(listordered, smalllist).ToList();
foreach (var item in listordered)
{
item.RangeIndex = lastrankindex++;
//ProcessTraceUtil.Trace("rangeindex:" + item.Key[0] + "->" + (item.RangeIndex));
}
ProcessTraceUtil.TraceMem("排序完成:" + listordered.Count + "条", "m");
}
if (listordered.Count > 0)
{
if (!isonlyoldlist)
{
ProcessTraceUtil.TraceMem("把排好的数据写入到新索引文件:" + listordered.Count + "条", "m");
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
foreach (var item in listordered)
{
item.KeyOffset = nw.GetWritePosition();
nw.AppendObject(item);
}
newIndexMergePos = nw.GetWritePosition();
}
ProcessTraceUtil.TraceMem("写入到新索引文件完成", "m");
}
if (listordered.Count <= 2 || loadFactor == 1)
{
newdiskindexlist.AddRange(listordered);
}
else
{
newdiskindexlist.Add(listordered.First());
int idx = 0;
foreach (var item in listordered)
{
if ((++idx) % loadFactor == 0)
{
newdiskindexlist.Add(item);
}
}
//newdiskindexlist.AddRange(listordered.Where(p => (++idx) % loadFactor == 0));
if (idx % loadFactor != 0)
{
newdiskindexlist.Add(listordered.Last());
}
}
}
ProcessTraceUtil.TraceMem("写入到新索引文件后整理索引完成", "m");
if (isall)
{
if (copylast > 0 && copylast < mergeinfo.IndexMergePos)
{
ProcessTraceUtil.TraceMem("copy已排序的大于新增最大数部分" + copylast + "->" + mergeinfo.IndexMergePos, "m");
var offset = 0L;
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
offset = nw.GetWritePosition() - copylast;
}
//copy
long newindexpos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, copylast, mergeinfo.IndexMergePos - 1, false);
foreach (var p in keyindexdisklist[tablename])
{
if (p.KeyOffset >= copylast && p.KeyOffset < mergeinfo.IndexMergePos)
{
newdiskindexlist.Add(new BigEntityTableIndexItem
{
Del = p.Del,
Key = p.Key,
KeyOffset = p.KeyOffset + offset,
len = p.len,
Offset = p.Offset,
Index = p.Index,
RangeIndex = p.RangeIndex + readcount
});
}
}
ProcessTraceUtil.TraceMem("copy数据完成->" + offset, "m");
}
break;
}
else
{
if (listtemp.Count > 0)
{
long newcopybefore = 0;
using (var sortarray = new Collections.SorteArray <BigEntityTableIndexItem>(indexarray))
{
int mid = -1;
int pos = sortarray.Find(listtemp.First(), ref mid);
ProcessTraceUtil.Trace("查找已经排序的小于新增最小数据部分:mid=" + mid + ",pos=" + pos);
if (pos == -1 && mid != -1)
{
//小于最小的
newcopybefore = indexarray[mid].KeyOffset;
lastrankindex = indexarray[mid].RangeIndex + readcount - listtemp.Count;
}
else if (pos != -1)
{
newcopybefore = indexarray[pos].KeyOffset;
lastrankindex = indexarray[pos].RangeIndex + readcount - listtemp.Count;
}
}
if (newcopybefore > readoldstartpostion)
{
ProcessTraceUtil.Trace("中间copy");
var offset = 0L;
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
offset = nw.GetWritePosition() - readoldstartpostion;
}
IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, readoldstartpostion, newcopybefore - 1, false);
foreach (var p in keyindexdisklist[tablename])
{
if (p.KeyOffset >= readoldstartpostion && p.KeyOffset < newcopybefore)
{
newdiskindexlist.Add(new BigEntityTableIndexItem
{
Del = p.Del,
Key = p.Key,
KeyOffset = p.KeyOffset + offset,
len = p.len,
Offset = p.Offset,
Index = p.Index,
RangeIndex = p.RangeIndex + readcount - listtemp.Count
});
}
}
readoldstartpostion = newcopybefore;
ProcessTraceUtil.Trace("中间copy完成");
}
else if (newcopybefore < readoldstartpostion)
{
ProcessTraceUtil.Trace("补中间");
reader.SetPostion(newcopybefore);
//补充数目
foreach (var item in reader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, smallbuffer))
{
item.KeyOffset = keyoffset;
item.SetIndex(meta.KeyIndexInfo);
//ProcessTraceUtil.Trace(item.Key[0].ToString());
if (item.KeyOffset >= mergeinfo.IndexMergePos)
{
break;
}
if (copylast > 0 && item.KeyOffset >= copylast)
{
break;
}
if (item.KeyOffset == readoldstartpostion)
{
break;
}
if (item.Del)
{
continue;
}
//ProcessTraceUtil.Trace("补中间+1");
lastrankindex++;
}
}
}
else
{
ProcessTraceUtil.Trace("中间copy2");
var offset = 0L;
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
offset = nw.GetWritePosition() - readoldstartpostion;
}
IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, readoldstartpostion, copylast - 1, false);
foreach (var p in keyindexdisklist[tablename])
{
if (p.KeyOffset >= readoldstartpostion && p.KeyOffset < copylast)
{
newdiskindexlist.Add(new BigEntityTableIndexItem
{
Del = p.Del,
Key = p.Key,
KeyOffset = p.KeyOffset + offset,
len = p.len,
Offset = p.Offset,
Index = p.Index,
RangeIndex = p.RangeIndex + readcount
});
}
}
readoldstartpostion = copylast;
ProcessTraceUtil.Trace("中间copy2完成");
}
}
}
}
//后面copy
string tablefile = GetTableFile(tablename);
var idxreader = ObjTextReader.CreateReader(indexfile);
var newwriter = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2);
try
{
long nextcopypos = 0;
ProcessTraceUtil.TraceMem("读取后面的数据->" + lasmargepos, "m");
idxreader.SetPostion(lasmargepos);
bool hasitem = false;
bool isfirst = true;
long keyoffset = 0;
foreach (var item in idxreader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, smallbuffer))
{
hasitem = true;
item.SetIndex(meta.KeyIndexInfo);
item.KeyOffset = keyoffset;
if (item.KeyOffset > newwriter.GetWritePosition())
{
var spacelen = item.KeyOffset - newwriter.GetWritePosition();
ProcessTraceUtil.Trace("没有对齐，尝试对齐，spacelen->" + spacelen);
if (spacelen % 3 == 0)
{
newwriter.FillSpace(spacelen / 3);
}
else
{
ProcessTraceUtil.Trace("对齐失败");
var ex = new Exception("无法对齐");
ex.Data.Add("老索引文件当前位置", item.KeyOffset);
ex.Data.Add("新索引文件写入位置", newwriter.GetWritePosition());
throw ex;
}
}
if (item.KeyOffset == newwriter.GetWritePosition())
{
nextcopypos = newwriter.GetWritePosition();
if (isfirst)
{
newIndexMergePos = nextcopypos;
}
ProcessTraceUtil.Trace("新老索引文件已经对齐:" + item.KeyOffset);
break;
}
isfirst = false;
item.KeyOffset = newwriter.GetWritePosition();
newwriter.AppendObject(item);
}
if (nextcopypos > 0)
{
newwriter.Dispose();
ProcessTraceUtil.Trace("copy后面的数据->" + nextcopypos);
nextcopypos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, nextcopypos, -512);
}
else if (!hasitem)
{
var idxpos = idxreader.ReadedPostion();
if (idxpos == newwriter.GetWritePosition())
{
nextcopypos = idxpos;
newwriter.Dispose();
}
else
{
ProcessTraceUtil.Trace(idxpos + " vs " + newwriter.GetWritePosition());
}
}
ProcessTraceUtil.TraceMem("读取后面的数据完成", "m");
if (nextcopypos == 0)
{
throw new Exception("更新索引出错");
}
try
{
BigEntityTableIndexItem[] oldindexarray = null;
BigEntityTableIndexItem[] newdiskindexarray = newdiskindexlist.ToArray();
tablelocker.EnterWriteLock();
ProcessTraceUtil.TraceMem("读取后面的数据->" + lasmargepos, "m");
if (nextcopypos <= 0)
{
using (newwriter)
{
using (idxreader)
{
foreach (var item in idxreader.ReadObjectsWating <BigEntityTableIndexItem>(1, bytes: bigbuffer))
{
item.SetIndex(meta.KeyIndexInfo);
item.KeyOffset = newwriter.GetWritePosition();
newwriter.AppendObject(item);
}
}
}
}
else
{
nextcopypos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, nextcopypos, long.MaxValue);
ProcessTraceUtil.TraceMem("继续copy后面的数据->" + nextcopypos, "m");
idxreader.Dispose();
}
ProcessTraceUtil.TraceMem("读取后面的数据完成", "m");
//更新索引
mergeinfo.LoadFactor = loadFactor;
keyindexdisklist.TryRemove(tablename, out oldindexarray);
keyindexdisklist.TryAdd(tablename, newdiskindexarray);
keyindexmemtemplist[tablename] = new SortArrayList <BigEntityTableIndexItem>();
int trycount = 0;
while (true)
{
try
{
File.Delete(indexfile);
ProcessTraceUtil.Trace("删除旧文件完成");
break;
}
catch (System.IO.IOException ex)
{
Thread.Sleep(1);
trycount++;
if (trycount > 1000)
{
throw ex;
}
}
}
File.Move(newindexfile, indexfile);
ProcessTraceUtil.TraceMem("删除旧文件，重命名新文件完成", "m");
idxreader = null;
}
finally
{
tablelocker.ExitWriteLock();
}
string metafile = GetMetaFile(tablename);
mergeinfo.IndexMergePos = lasmargepos;
SerializerHelper.SerializerToXML(meta, metafile, true);
ProcessTraceUtil.Trace("更新元文件，更新索引完成");
}
catch (Exception ex)
{
Console.WriteLine("整理索引后面部分出错" + ex.ToString());
//LogManager.LogHelper.Instance.Error("整理索引后面部分出错", ex);
}
finally
{
if (idxreader != null)
{
idxreader.Dispose();
}
if (newwriter != null && !newwriter.Isdispose)
{
newwriter.Dispose();
}
if (File.Exists(newindexfile))
{
File.Delete(newindexfile);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
//LogManager.LogHelper.Instance.Error("整理索引出错", ex);
}
finally
{
//GC.Collect();
//ProcessTraceUtil.TraceMem("回收内存","m");
mergeinfo.IsMergin = false;
var info = ProcessTraceUtil.PrintTrace();
Console.WriteLine(info);
//LogManager.LogHelper.Instance.Debug("整理索引过程:" + info);
}
}
private void LoadKey(string tablename, BigEntityTableMeta meta)
{
string indexfile = GetKeyFile(tablename);
var indexmergeinfo = meta.IndexMergeInfos.Find(p => p.IndexName.Equals(meta.KeyName));
if (indexmergeinfo == null)
{
indexmergeinfo = new IndexMergeInfo();
indexmergeinfo.IndexName = meta.KeyName;
meta.IndexMergeInfos.Add(indexmergeinfo);
}
//计算加载因子
indexmergeinfo.LoadFactor = (int)Math.Max(4, new FileInfo(indexfile).Length / MAX_KEYBUFFER);
int i = 0;
BigEntityTableIndexItem lastreadindex = null;
List <BigEntityTableIndexItem> list = new List <BigEntityTableIndexItem>();
long currentpos = 0;
long currrankindex = 0;
byte[] buffer = new byte[1024 * 1024 * 10];
using (ObjTextReader idx = ObjTextReader.CreateReader(indexfile))
{
Console.WriteLine("loadkey");
foreach (var newindex in idx.ReadObjectsWating <BigEntityTableIndexItem>(1, p => currentpos = p, buffer))
{
if (newindex.Del)
{
continue;
}
newindex.KeyOffset = currentpos;
newindex.RangeIndex = currrankindex++;
newindex.SetIndex(meta.KeyIndexInfo);
if (newindex.KeyOffset >= indexmergeinfo.IndexMergePos)
{
//list.Add(newindex);
if (list.Count > 0)
{
if (list.Last().KeyOffset != lastreadindex.KeyOffset)
{
list.Add(lastreadindex);
}
}
break;
}
if (indexmergeinfo.LoadFactor == 1 || i % indexmergeinfo.LoadFactor == 0)
{
list.Add(newindex);
}
i++;
lastreadindex = newindex;
}
if (list.Count > 0 && list.Last().KeyOffset != lastreadindex.KeyOffset)
{
list.Add(lastreadindex);
}
}
indexmergeinfo.TotalCount = i;
BigEntityTableIndexItem[] oldindexitems = null;
keyindexdisklist.TryRemove(tablename, out oldindexitems);
keyindexdisklist.TryAdd(tablename, list.ToArray());
using (ObjTextReader idr = ObjTextReader.CreateReader(indexfile))
{
Console.WriteLine("loadkey2");
if (indexmergeinfo.IndexMergePos > 0)
{
idr.SetPostion(indexmergeinfo.IndexMergePos);
}
//Dictionary<string, BigEntityTableIndexItem> indexdic = new Dictionary<string, BigEntityTableIndexItem>();
//keyindexlistdic[tablename];
SortArrayList <BigEntityTableIndexItem> keymemlist = new SortArrayList <BigEntityTableIndexItem>();
foreach (var newindex in idr.ReadObjectsWating <BigEntityTableIndexItem>(1, p => currentpos = p, buffer))
{
newindex.KeyOffset = currentpos;
newindex.SetIndex(meta.KeyIndexInfo);
newindex.RangeIndex = -1;
if (!newindex.Del)
{
//indexdic.Add(newindex.Key, newindex);
keymemlist.Add(newindex);
i++;
}
}
var tablelocker = GetKeyLocker(tablename, string.Empty);
tablelocker.EnterWriteLock();
try
{
foreach (var newindex in idr.ReadObjectsWating <BigEntityTableIndexItem>(1, p => currentpos = p, buffer))
{
newindex.KeyOffset = currentpos;
newindex.SetIndex(meta.KeyIndexInfo);
newindex.RangeIndex = -1;
if (!newindex.Del)
{
//indexdic.Add(newindex.Key, newindex);
keymemlist.Add(newindex);
i++;
}
}
if (idr.Length() - currentpos > 10240)
{
throw new Exception(tablename + "主键索引大量数据未读取");
}
//keyindexlistdic[tablename] = indexdic;
keyindexmemlist[tablename] = keymemlist;
}
finally
{
tablelocker.ExitWriteLock();
}
}
}
public LocalFileQueue(string queuename, string queuefilepath, bool canwrite = true, bool canread = true)
{
if (string.IsNullOrWhiteSpace(queuename))
{
throw new ArgumentNullException("queuename");
}
QueueName = queuename;
if (string.IsNullOrWhiteSpace(queuefilepath))
{
throw new ArgumentNullException("queuefilepath");
}
QueueFilePath = queuefilepath;
if (!File.Exists(queuefilepath))
{
FileInfo file = new FileInfo(queuefilepath);
if (!file.Directory.Exists)
{
try
{
file.Directory.Create();
}
catch (Exception ex)
{
throw new Exception("创建文件夹失败:" + file.Directory.FullName, ex);
}
}
}
if (canwrite)
{
//由于文件写是独占式的，当程序快速重启时，文件可能未及时释放
int trytimes = 0;
while (true)
{
try
{
//_queueWriter = ObjTextWriter.CreateWriter(queuefilepath, ObjTextReaderWriterEncodeType.jsonbuf);
_queueWriter = ObjTextWriter.CreateWriter(queuefilepath, ObjTextReaderWriterEncodeType.entitybuf);
_queueWriter.Flush();
break;
}
catch (Exception ex)
{
trytimes++;
if (trytimes >= 3)
{
throw ex;
}
Thread.Sleep(1000 * trytimes);
}
}
}
if (canread)
{
while (true)
{
if (File.Exists(queuefilepath))
{
break;
}
Thread.Sleep(1000);
}
_queueReader = ObjTextReader.CreateReader(queuefilepath);
FileInfo finfo = new FileInfo(queuefilepath);
QueueCfgFile = finfo.Directory.FullName + "\\" + queuename + ".cfg";
if (File.Exists(QueueCfgFile))
{
_logger = LJC.FrameWork.Comm.SerializerHelper.DeSerializerFile <LocalFileQueueCfg>(QueueCfgFile, true);
if (_logger.LastPos > 0)
{
_queueReader.SetPostion(_logger.LastPos);
}
}
else
{
_logger = new LocalFileQueueCfg();
_logger.LastChageTime = DateTime.Now;
_logger.QueueFile = queuefilepath;
SaveConfig();
}
}
_backtimer = new Timer(new TimerCallback(TimerAction), null, 0, 0);
}