static void TestRWObjEx()
{
int i = 0;
string filename = "testrwobjex.bin";
using (LJC.FrameWork.Comm.ObjTextWriter writer = ObjTextWriter.CreateWriter(filename, ObjTextReaderWriterEncodeType.protobufex))
{
while ((i++) < 100)
{
writer.AppendObject <Man>(new Man
{
Name = "李金川" + i,
IDCard = "421182198612301310",
Addr = "湖北省武穴市",
Sex = 1
});
//writer.Flush();
Console.WriteLine("写入成功一条！" + i);
Thread.Sleep(10);
}
}
}
static void TestRWObj1()
{
int i = 0;
string filename = "testrwobj1.bin";
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
using (LJC.FrameWork.Comm.ObjTextWriter writer = ObjTextWriter.CreateWriter(filename, ObjTextReaderWriterEncodeType.protobufex))
{
while ((i++) < 1000000)
{
writer.AppendObject <Man>(new Man
{
Name = "李金川" + i,
IDCard = "421182198612301310",
Addr = "湖北省武穴市",
Sex = 1
});
//writer.Flush();
if (i % 10000 == 0)
{
Console.WriteLine("写入成功到" + i);
}
//Thread.Sleep(1000);
}
}
sw.Stop();
Console.WriteLine("共用时：" + sw.Elapsed.TotalSeconds + "秒");
}
public bool Delete(string tablename, string key)
{
EntityTableMeta meta = GetMetaData(tablename);
Dictionary <long, EntityTableIndexItem> arr = null;
var locker = GetKeyLocker(tablename, string.Empty);
lock (locker)
{
if (keyindexdic[tablename].TryRemove(key, out arr))
{
string keyindexfile = GetKeyFile(tablename);
using (ObjTextWriter keywriter = ObjTextWriter.CreateWriter(keyindexfile, ObjTextReaderWriterEncodeType.entitybuf))
//ObjTextWriter keywriter = GetWriter(keyindexfile);
//lock(keywriter)
{
foreach (var item in arr)
{
var indexitem = (EntityTableIndexItem)item.Value;
indexitem.Del = true;
keywriter.AppendObject(indexitem);
foreach (var idx in meta.Indexs)
{
string indexfile = GetIndexFile(tablename, idx);
using (ObjTextWriter idxwriter = ObjTextWriter.CreateWriter(indexfile, ObjTextReaderWriterEncodeType.entitybuf))
//ObjTextWriter idxwriter = GetWriter(indexfile);
//lock(idxwriter)
{
idxwriter.AppendObject(indexitem);
}
}
}
}
return(true);
}
}
return(false);
}
private ObjTextWriter GetWriter(string filename, int locksecs = 1)
{
ObjTextWriter writer = null;
if (writerdic.TryGetValue(filename, out writer))
{
lock (writer)
{
if (!writer.Isdispose)
{
writer.Tag = DateTime.Now;
return(writer);
}
}
}
lock (writerdic)
{
if (writerdic.TryGetValue(filename, out writer))
{
lock (writer)
{
if (!writer.Isdispose)
{
writer.Tag = DateTime.Now;
return(writer);
}
}
}
writer = ObjTextWriter.CreateWriter(filename, ObjTextReaderWriterEncodeType.entitybuf);
writer.Tag = DateTime.Now;
writerdic.Add(filename, writer);
LJC.FrameWork.Comm.Coroutine.CoroutineEngine.DefaultCoroutineEngine.Dispatcher(new WriterDestroy(writerdic, filename, locksecs));
}
return(writer);
}
private bool Update2 <T>(string tablename, string key, T item, EntityTableMeta meta) where T : new()
{
string tablefile = GetTableFile(tablename);
Dictionary <long, EntityTableIndexItem> arr = null;
Tuple <long, long> offset = null;
int indexpos = 0;
var locker = GetKeyLocker(tablename, string.Empty);
lock (locker)
{
if (!keyindexdic[tablename].TryGetValue(key, out arr))
{
throw new Exception(string.Format("更新失败，key为{0}的记录数为0", key));
}
string keyindexfile = GetKeyFile(tablename);
EntityTableIndexItem indexitem = (EntityTableIndexItem)arr.Last().Value;
using (ObjTextWriter keywriter = ObjTextWriter.CreateWriter(keyindexfile, ObjTextReaderWriterEncodeType.entitybuf))
//ObjTextWriter keywriter = GetWriter(keyindexfile);
{
indexitem.Del = true;
keywriter.AppendObject(indexitem);
}
var keyvalue = item.Eval(meta.KeyProperty);
//var keyvalue=meta.KeyProperty.GetValueMethed(item);
if (keyvalue == null)
{
throw new Exception("key值不能为空");
}
var tableitem = new EntityTableItem <T>(item);
tableitem.Flag = (byte)EntityTableItemFlag.Ok;
using (ObjTextWriter otw = ObjTextWriter.CreateWriter(tablefile, ObjTextReaderWriterEncodeType.entitybuf))
//ObjTextWriter otw = GetWriter(tablefile);
{
offset = otw.PreAppendObject(tableitem, (s1, s2) =>
{
if (s1.Length <= indexitem.len)
{
Console.WriteLine("修改->覆盖");
return(otw.Override(indexitem.Offset, s1));
}
return(null);
});
}
Dictionary <long, EntityTableIndexItem> al = null;
var keyidc = keyindexdic[tablename];
if (!keyidc.TryGetValue(keyvalue.ToString(), out al))
{
lock (keyidc)
{
if (!keyidc.TryGetValue(keyvalue.ToString(), out al))
{
al = new Dictionary <long, EntityTableIndexItem>();
keyidc.TryAdd(keyvalue.ToString(), al);
}
}
}
EntityTableIndexItem newkey = new EntityTableIndexItem
{
Key = keyvalue.ToString(),
Offset = offset.Item1,
len = (int)(offset.Item2 - offset.Item1),
Del = false
};
if (newkey.Offset == indexitem.Offset)
{
newkey.len = indexitem.len;
}
arr[indexpos] = newkey;
using (ObjTextWriter keyidxwriter = ObjTextWriter.CreateWriter(keyindexfile, ObjTextReaderWriterEncodeType.entitybuf))
//ObjTextWriter keyidxwriter = GetWriter(keyindexfile);
//lock(keyidxwriter)
{
keyidxwriter.AppendObject(newkey);
}
foreach (var idx in meta.Indexs)
{
string indexfile = GetIndexFile(tablename, idx);
var indexvalue = item.Eval(meta.IndexProperties[idx]);
var newindex = new EntityTableIndexItem
{
Key = indexvalue == null ? string.Empty : indexvalue.ToString(),
Offset = offset.Item1,
len = (int)(offset.Item2 - offset.Item1),
Del = false
};
using (ObjTextWriter idxwriter = ObjTextWriter.CreateWriter(indexfile, ObjTextReaderWriterEncodeType.entitybuf))
//ObjTextWriter idxwriter = GetWriter(indexfile);
//lock(idxwriter)
{
idxwriter.AppendObject(newindex);
}
}
Console.WriteLine("写入成功:" + keyvalue + "->" + offset);
}
return(true);
}
private bool Insert2 <T>(string tablename, IEnumerable <T> items, EntityTableMeta meta) where T : new()
{
string tablefile = GetTableFile(tablename);
var locker = GetKeyLocker(tablename, string.Empty);
lock (locker)
{
ObjTextWriter otw = ObjTextWriter.CreateWriter(tablefile, ObjTextReaderWriterEncodeType.entitybuf);
string keyindexfile = GetKeyFile(tablename);
ObjTextWriter keywriter = ObjTextWriter.CreateWriter(keyindexfile, ObjTextReaderWriterEncodeType.entitybuf);
Dictionary <string, ObjTextWriter> idxwriterdic = new Dictionary <string, ObjTextWriter>();
try
{
foreach (var item in items)
{
var keyvalue = item.Eval(meta.KeyProperty);
//var keyvalue = meta.KeyProperty.GetValueMethed(item);
if (keyvalue == null)
{
throw new Exception("key值不能为空");
}
var keystr = keyvalue.ToString();
var keylocker = GetKeyLocker(tablename, keystr);
Dictionary <long, EntityTableIndexItem> arr = null;
lock (keylocker)
{
if (keyindexdic[tablename].TryGetValue(keystr, out arr))
{
throw new Exception(string.Format("key:{0}不可重复", keystr));
}
}
var tableitem = new EntityTableItem <T>(item);
tableitem.Flag = (byte)EntityTableItemFlag.Ok;
//using (ObjTextWriter otw = ObjTextWriter.CreateWriter(tablefile, ObjTextReaderWriterEncodeType.entitybuf))
//var otw = GetWriter(tablefile);
//lock(otw)
{
var offset = otw.AppendObject(tableitem);
Dictionary <long, EntityTableIndexItem> al = null;
var idc = keyindexdic[tablename];
if (!idc.TryGetValue(keystr, out al))
{
lock (idc)
{
if (!idc.TryGetValue(keystr, out al))
{
al = new Dictionary <long, EntityTableIndexItem>();
idc.TryAdd(keystr, al);
}
}
}
var newkey = new EntityTableIndexItem
{
Key = keystr,
Offset = offset.Item1,
len = (int)(offset.Item2 - offset.Item1)
};
lock (al)
{
al.Add(newkey.Offset, newkey);
}
//string keyindexfile = GetKeyFile(tablename);
//using (ObjTextWriter keywriter = ObjTextWriter.CreateWriter(keyindexfile, ObjTextReaderWriterEncodeType.entitybuf))
//ObjTextWriter keywriter = GetWriter(keyindexfile);
//lock(keywriter)
{
keywriter.AppendObject(newkey);
}
foreach (var idx in meta.Indexs)
{
string indexfile = GetIndexFile(tablename, idx);
var indexvalue = item.Eval(meta.IndexProperties[idx]);
var newindex = new EntityTableIndexItem
{
Key = indexvalue == null ? string.Empty : indexvalue.ToString(),
Offset = offset.Item1,
len = (int)(offset.Item2 - offset.Item1)
};
//using (ObjTextWriter idxwriter = ObjTextWriter.CreateWriter(indexfile, ObjTextReaderWriterEncodeType.entitybuf))
//ObjTextWriter idxwriter = GetWriter(indexfile);
//lock(idxwriter)
{
ObjTextWriter idxwriter = null;
if (!idxwriterdic.TryGetValue(indexfile, out idxwriter))
{
idxwriter = ObjTextWriter.CreateWriter(indexfile, ObjTextReaderWriterEncodeType.entitybuf);
idxwriterdic.Add(indexfile, idxwriter);
}
idxwriter.AppendObject(newindex);
}
}
}
}
}
finally
{
otw.Dispose();
keywriter.Dispose();
foreach (var kv in idxwriterdic)
{
kv.Value.Dispose();
}
}
}
return(true);
}
public void CreateTable(string tablename, string keyname, Type ttype, string[] indexs = null)
{
string tablefile = GetTableFile(tablename);
bool delfile = true;
if (!File.Exists(tablefile))
{
try
{
using (ObjTextWriter otw = ObjTextWriter.CreateWriter(tablefile, ObjTextReaderWriterEncodeType.entitybuf))
//ObjTextWriter otw = GetWriter(tablefile);
//lock(otw)
{
string metafile = GetMetaFile(tablename);
if (!File.Exists(metafile))
{
EntityTableMeta meta = new EntityTableMeta();
meta.KeyName = keyname;
meta.Indexs = indexs ?? new string[] { };
meta.CTime = DateTime.Now;
meta.TType = ttype;
var pp = ReflectionHelper.GetProperty(ttype, keyname);
if (pp == null)
{
throw new Exception("找不到主键:" + keyname);
}
meta.KeyProperty = new PropertyInfoEx(pp);
if (indexs != null && indexs.Length > 0)
{
if (indexs.Contains(keyname))
{
throw new Exception("索引不能包含主键");
}
indexs = indexs.Distinct().ToArray();
foreach (var idx in indexs)
{
var idxpp = ReflectionHelper.GetProperty(ttype, idx);
if (idxpp == null)
{
throw new Exception("找不到索引:" + idx);
}
if (!meta.IndexProperties.ContainsKey(idx))
{
meta.IndexProperties.Add(idx, new PropertyInfoEx(idxpp));
}
}
}
LJC.FrameWork.Comm.SerializerHelper.SerializerToXML <EntityTableMeta>(meta, metafile, catchErr: true);
metadic.Add(tablename, meta);
keyindexdic.TryAdd(tablename, new ConcurrentDictionary <string, Dictionary <long, EntityTableIndexItem> >());
}
else
{
var meta = LJC.FrameWork.Comm.SerializerHelper.DeSerializerFile <EntityTableMeta>(metafile, true);
if (!meta.KeyName.Equals(keyname) || !meta.TypeString.Equals(ttype))
{
delfile = false;
throw new Exception("meta文件内容检查不一致");
}
}
//索引
string keyfile = GetKeyFile(tablename);
using (ObjTextWriter keyidxwriter = ObjTextWriter.CreateWriter(keyfile, ObjTextReaderWriterEncodeType.entitybuf))
//ObjTextWriter keyidxwriter = GetWriter(keyfile);
{
}
if (indexs != null)
{
foreach (var idx in indexs)
{
var indexfile = GetIndexFile(tablename, idx);
using (ObjTextWriter idxwriter = ObjTextWriter.CreateWriter(indexfile, ObjTextReaderWriterEncodeType.entitybuf))
//ObjTextWriter idxwriter = GetWriter(indexfile);
{
}
}
}
}
}
catch (Exception ex)
{
if (delfile && File.Exists(tablefile))
{
File.Delete(tablefile);
}
throw ex;
}
}
}
public void MergeKey2(string tablename, string indexname, BigEntityTableMeta meta)
{
ProcessTraceUtil.StartTrace();
IndexMergeInfo mergeinfo = null;
lock (meta)
{
mergeinfo = meta.IndexMergeInfos.Find(p => indexname.Equals(p.IndexName));
if (mergeinfo == null)
{
mergeinfo = new IndexMergeInfo();
mergeinfo.IndexName = indexname;
meta.IndexMergeInfos.Add(mergeinfo);
}
if (mergeinfo.IsMergin)
{
return;
}
meta.NewAddCount = 0;
mergeinfo.IsMergin = true;
}
DateTime timestart = DateTime.Now;
string newindexfile = string.Empty;
byte[] bigbuffer = new byte[1024 * 1024];
byte[] smallbuffer = new byte[2048];
try
{
ProcessTraceUtil.TraceMem("开始整理key:" + indexname, "m");
long lasmargepos = 0;
long newIndexMergePos = mergeinfo.IndexMergePos;
string indexfile = indexname.Equals(meta.KeyName) ? GetKeyFile(tablename) : GetIndexFile(tablename, indexname);
var tablelocker = GetKeyLocker(tablename, string.Empty);
//新的磁盘索引
List <BigEntityTableIndexItem> newdiskindexlist = new List <BigEntityTableIndexItem>();
var loadFactor = (int)Math.Max(4, new FileInfo(indexfile).Length / MAX_KEYBUFFER);
using (var reader = ObjTextReader.CreateReader(indexfile))
{
long readoldstartpostion = reader.ReadedPostion();
try
{
tablelocker.EnterWriteLock();
keyindexmemtemplist[tablename] = keyindexmemlist[tablename];
keyindexmemlist[tablename] = new SortArrayList <BigEntityTableIndexItem>();
}
finally
{
tablelocker.ExitWriteLock();
}
var listtemp = keyindexmemtemplist[tablename].GetList().ToList();
listtemp = listtemp.Select(p => new BigEntityTableIndexItem
{
Del = p.Del,
Key = p.Key,
KeyOffset = p.KeyOffset,
len = p.len,
Offset = p.Offset,
Index = p.Index,
RangeIndex = p.RangeIndex
}).ToList();
int readcount = listtemp.Count;
ProcessTraceUtil.TraceMem("从内存中读取新增数据，共" + readcount + "条", "m");
if (readcount == 0)
{
return;
}
lasmargepos = listtemp.Max(p => p.KeyOffset);
reader.SetPostion(lasmargepos);
reader.ReadObject <BigEntityTableIndexItem>();
lasmargepos = reader.ReadedPostion();
//优化确定哪些部分是不需要一个个读入的
long copybefore = 0, copylast = 0;
long lastrankindex = 0;
var indexarray = keyindexdisklist[tablename];
using (var sortarray = new Collections.SorteArray <BigEntityTableIndexItem>(indexarray))
{
int mid = -1;
int pos = sortarray.Find(listtemp.First(), ref mid);
ProcessTraceUtil.Trace("查找新数据在老数中插入的开始位置:mid=" + mid + ",pos=" + pos);
if (pos == -1 && mid != -1)
{
//小于最小的
copybefore = indexarray[mid].KeyOffset;
lastrankindex = indexarray[mid].RangeIndex;
ProcessTraceUtil.Trace("老数据可以直接copy的部分:0->" + copybefore);
}
else if (pos != -1)
{
copybefore = indexarray[pos].KeyOffset;
lastrankindex = indexarray[pos].RangeIndex;
ProcessTraceUtil.Trace("老数据可以直接copy的部分:0->" + copybefore);
}
//优化确定后面读到哪
mid = -1;
pos = sortarray.Find(listtemp.Last(), ref mid);
ProcessTraceUtil.Trace("查找新数据在老数据中插入的结束位置:mid=" + mid + ",pos=" + pos);
if (pos == -1 && mid != -1 && mid < indexarray.Length - 1)
{
//小于最小的
copylast = indexarray[mid + 1].KeyOffset;
ProcessTraceUtil.Trace("老数据可以直接copy的部分:" + copylast + "->" + mergeinfo.IndexMergePos);
}
else if (pos != -1)
{
copylast = indexarray[pos].KeyOffset;
ProcessTraceUtil.Trace("老数据可以直接copy的部分:" + copylast + "->" + mergeinfo.IndexMergePos);
}
}
newindexfile = (indexname.Equals(meta.KeyName) ? GetKeyFile(tablename) : GetIndexFile(tablename, indexname)) + ".temp";
if (File.Exists(newindexfile))
{
File.Delete(newindexfile);
}
//快速copy
if (copybefore > 0)
{
ProcessTraceUtil.TraceMem("直接copy前面不在排序范围的数据:0->" + copybefore, "m");
IOUtil.CopyFile(indexfile, newindexfile, FileMode.Create, 0, copybefore - 1);
readoldstartpostion = copybefore;
ProcessTraceUtil.TraceMem("copy数据完成", "m");
newdiskindexlist.AddRange(keyindexdisklist[tablename].Where(p => p.KeyOffset < copybefore));
}
bool isall = false;
ProcessTraceUtil.TraceMem("开始读取在排序范围内的数据", "m");
while (true)
{
ProcessTraceUtil.Trace("读取老数据,开始位置:" + readoldstartpostion);
reader.SetPostion(readoldstartpostion);
var listordered = new List <BigEntityTableIndexItem>();
var loadcount = 0;
long keyoffset = 0;
foreach (var item in reader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, bigbuffer))
{
item.KeyOffset = keyoffset;
item.SetIndex(meta.KeyIndexInfo);
if (item.KeyOffset >= mergeinfo.IndexMergePos)
{
break;
}
if (copylast > 0 && item.KeyOffset >= copylast)
{
break;
}
listordered.Add(item);
if (++loadcount >= MERGE_TRIGGER_NEW_COUNT)
{
break;
}
}
readoldstartpostion = reader.ReadedPostion();
bool isonlyoldlist = false;
if (listordered.Count == 0)
{
ProcessTraceUtil.TraceMem("老数据没有了，全部是新数据:" + listtemp.Count, "m");
listordered = MergeAndSort2(listordered, listtemp).ToList();
foreach (var item in listordered)
{
item.RangeIndex = lastrankindex++;
}
isall = true;
}
else if (listtemp.Count == 0 && listordered.Count > 10000)
{
ProcessTraceUtil.TraceMem("新数据没有了，全部是老数据:" + listordered.Count, "m");
//copy
IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, listordered.First().KeyOffset, listordered.Last().KeyOffset - 1, false);
var listorderedlast = listordered.Last();
long copyoffset = 0;
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
var item = new BigEntityTableIndexItem {
Del = listorderedlast.Del, Key = listorderedlast.Key, len = listorderedlast.len, Offset = listorderedlast.Offset, Index = listorderedlast.Index
};
item.KeyOffset = nw.GetWritePosition();
copyoffset = nw.GetWritePosition() - listorderedlast.KeyOffset;
nw.AppendObject(item);
newIndexMergePos = nw.GetWritePosition();
}
isonlyoldlist = true;
//更新索引
foreach (var item in listordered)
{
item.RangeIndex = lastrankindex++;
item.KeyOffset += copyoffset;
}
ProcessTraceUtil.TraceMem("直接copy数据完成", "m");
}
else
{
ProcessTraceUtil.TraceMem("老数据条数为:" + listordered.Count + "，新数据条数为:" + listtemp.Count, "m");
var listordermax = listordered.Last();
int idx = 0;
foreach (var item in listtemp)
{
if (item.CompareTo(listordermax) > 0)
{
break;
}
else
{
idx++;
}
}
List <BigEntityTableIndexItem> smalllist = listtemp.Take(idx).ToList();
listtemp = listtemp.Skip(idx).ToList();
listordered = MergeAndSort2(listordered, smalllist).ToList();
foreach (var item in listordered)
{
item.RangeIndex = lastrankindex++;
//ProcessTraceUtil.Trace("rangeindex:" + item.Key[0] + "->" + (item.RangeIndex));
}
ProcessTraceUtil.TraceMem("排序完成:" + listordered.Count + "条", "m");
}
if (listordered.Count > 0)
{
if (!isonlyoldlist)
{
ProcessTraceUtil.TraceMem("把排好的数据写入到新索引文件:" + listordered.Count + "条", "m");
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
foreach (var item in listordered)
{
item.KeyOffset = nw.GetWritePosition();
nw.AppendObject(item);
}
newIndexMergePos = nw.GetWritePosition();
}
ProcessTraceUtil.TraceMem("写入到新索引文件完成", "m");
}
if (listordered.Count <= 2 || loadFactor == 1)
{
newdiskindexlist.AddRange(listordered);
}
else
{
newdiskindexlist.Add(listordered.First());
int idx = 0;
foreach (var item in listordered)
{
if ((++idx) % loadFactor == 0)
{
newdiskindexlist.Add(item);
}
}
//newdiskindexlist.AddRange(listordered.Where(p => (++idx) % loadFactor == 0));
if (idx % loadFactor != 0)
{
newdiskindexlist.Add(listordered.Last());
}
}
}
ProcessTraceUtil.TraceMem("写入到新索引文件后整理索引完成", "m");
if (isall)
{
if (copylast > 0 && copylast < mergeinfo.IndexMergePos)
{
ProcessTraceUtil.TraceMem("copy已排序的大于新增最大数部分" + copylast + "->" + mergeinfo.IndexMergePos, "m");
var offset = 0L;
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
offset = nw.GetWritePosition() - copylast;
}
//copy
long newindexpos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, copylast, mergeinfo.IndexMergePos - 1, false);
foreach (var p in keyindexdisklist[tablename])
{
if (p.KeyOffset >= copylast && p.KeyOffset < mergeinfo.IndexMergePos)
{
newdiskindexlist.Add(new BigEntityTableIndexItem
{
Del = p.Del,
Key = p.Key,
KeyOffset = p.KeyOffset + offset,
len = p.len,
Offset = p.Offset,
Index = p.Index,
RangeIndex = p.RangeIndex + readcount
});
}
}
ProcessTraceUtil.TraceMem("copy数据完成->" + offset, "m");
}
break;
}
else
{
if (listtemp.Count > 0)
{
long newcopybefore = 0;
using (var sortarray = new Collections.SorteArray <BigEntityTableIndexItem>(indexarray))
{
int mid = -1;
int pos = sortarray.Find(listtemp.First(), ref mid);
ProcessTraceUtil.Trace("查找已经排序的小于新增最小数据部分:mid=" + mid + ",pos=" + pos);
if (pos == -1 && mid != -1)
{
//小于最小的
newcopybefore = indexarray[mid].KeyOffset;
lastrankindex = indexarray[mid].RangeIndex + readcount - listtemp.Count;
}
else if (pos != -1)
{
newcopybefore = indexarray[pos].KeyOffset;
lastrankindex = indexarray[pos].RangeIndex + readcount - listtemp.Count;
}
}
if (newcopybefore > readoldstartpostion)
{
ProcessTraceUtil.Trace("中间copy");
var offset = 0L;
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
offset = nw.GetWritePosition() - readoldstartpostion;
}
IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, readoldstartpostion, newcopybefore - 1, false);
foreach (var p in keyindexdisklist[tablename])
{
if (p.KeyOffset >= readoldstartpostion && p.KeyOffset < newcopybefore)
{
newdiskindexlist.Add(new BigEntityTableIndexItem
{
Del = p.Del,
Key = p.Key,
KeyOffset = p.KeyOffset + offset,
len = p.len,
Offset = p.Offset,
Index = p.Index,
RangeIndex = p.RangeIndex + readcount - listtemp.Count
});
}
}
readoldstartpostion = newcopybefore;
ProcessTraceUtil.Trace("中间copy完成");
}
else if (newcopybefore < readoldstartpostion)
{
ProcessTraceUtil.Trace("补中间");
reader.SetPostion(newcopybefore);
//补充数目
foreach (var item in reader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, smallbuffer))
{
item.KeyOffset = keyoffset;
item.SetIndex(meta.KeyIndexInfo);
//ProcessTraceUtil.Trace(item.Key[0].ToString());
if (item.KeyOffset >= mergeinfo.IndexMergePos)
{
break;
}
if (copylast > 0 && item.KeyOffset >= copylast)
{
break;
}
if (item.KeyOffset == readoldstartpostion)
{
break;
}
if (item.Del)
{
continue;
}
//ProcessTraceUtil.Trace("补中间+1");
lastrankindex++;
}
}
}
else
{
ProcessTraceUtil.Trace("中间copy2");
var offset = 0L;
using (var nw = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2))
{
offset = nw.GetWritePosition() - readoldstartpostion;
}
IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, readoldstartpostion, copylast - 1, false);
foreach (var p in keyindexdisklist[tablename])
{
if (p.KeyOffset >= readoldstartpostion && p.KeyOffset < copylast)
{
newdiskindexlist.Add(new BigEntityTableIndexItem
{
Del = p.Del,
Key = p.Key,
KeyOffset = p.KeyOffset + offset,
len = p.len,
Offset = p.Offset,
Index = p.Index,
RangeIndex = p.RangeIndex + readcount
});
}
}
readoldstartpostion = copylast;
ProcessTraceUtil.Trace("中间copy2完成");
}
}
}
}
//后面copy
string tablefile = GetTableFile(tablename);
var idxreader = ObjTextReader.CreateReader(indexfile);
var newwriter = ObjTextWriter.CreateWriter(newindexfile, ObjTextReaderWriterEncodeType.entitybuf2);
try
{
long nextcopypos = 0;
ProcessTraceUtil.TraceMem("读取后面的数据->" + lasmargepos, "m");
idxreader.SetPostion(lasmargepos);
bool hasitem = false;
bool isfirst = true;
long keyoffset = 0;
foreach (var item in idxreader.ReadObjectsWating <BigEntityTableIndexItem>(1, p => keyoffset = p, smallbuffer))
{
hasitem = true;
item.SetIndex(meta.KeyIndexInfo);
item.KeyOffset = keyoffset;
if (item.KeyOffset > newwriter.GetWritePosition())
{
var spacelen = item.KeyOffset - newwriter.GetWritePosition();
ProcessTraceUtil.Trace("没有对齐，尝试对齐，spacelen->" + spacelen);
if (spacelen % 3 == 0)
{
newwriter.FillSpace(spacelen / 3);
}
else
{
ProcessTraceUtil.Trace("对齐失败");
var ex = new Exception("无法对齐");
ex.Data.Add("老索引文件当前位置", item.KeyOffset);
ex.Data.Add("新索引文件写入位置", newwriter.GetWritePosition());
throw ex;
}
}
if (item.KeyOffset == newwriter.GetWritePosition())
{
nextcopypos = newwriter.GetWritePosition();
if (isfirst)
{
newIndexMergePos = nextcopypos;
}
ProcessTraceUtil.Trace("新老索引文件已经对齐:" + item.KeyOffset);
break;
}
isfirst = false;
item.KeyOffset = newwriter.GetWritePosition();
newwriter.AppendObject(item);
}
if (nextcopypos > 0)
{
newwriter.Dispose();
ProcessTraceUtil.Trace("copy后面的数据->" + nextcopypos);
nextcopypos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, nextcopypos, -512);
}
else if (!hasitem)
{
var idxpos = idxreader.ReadedPostion();
if (idxpos == newwriter.GetWritePosition())
{
nextcopypos = idxpos;
newwriter.Dispose();
}
else
{
ProcessTraceUtil.Trace(idxpos + " vs " + newwriter.GetWritePosition());
}
}
ProcessTraceUtil.TraceMem("读取后面的数据完成", "m");
if (nextcopypos == 0)
{
throw new Exception("更新索引出错");
}
try
{
BigEntityTableIndexItem[] oldindexarray = null;
BigEntityTableIndexItem[] newdiskindexarray = newdiskindexlist.ToArray();
tablelocker.EnterWriteLock();
ProcessTraceUtil.TraceMem("读取后面的数据->" + lasmargepos, "m");
if (nextcopypos <= 0)
{
using (newwriter)
{
using (idxreader)
{
foreach (var item in idxreader.ReadObjectsWating <BigEntityTableIndexItem>(1, bytes: bigbuffer))
{
item.SetIndex(meta.KeyIndexInfo);
item.KeyOffset = newwriter.GetWritePosition();
newwriter.AppendObject(item);
}
}
}
}
else
{
nextcopypos = IOUtil.CopyFile(indexfile, newindexfile, FileMode.Open, nextcopypos, long.MaxValue);
ProcessTraceUtil.TraceMem("继续copy后面的数据->" + nextcopypos, "m");
idxreader.Dispose();
}
ProcessTraceUtil.TraceMem("读取后面的数据完成", "m");
//更新索引
mergeinfo.LoadFactor = loadFactor;
keyindexdisklist.TryRemove(tablename, out oldindexarray);
keyindexdisklist.TryAdd(tablename, newdiskindexarray);
keyindexmemtemplist[tablename] = new SortArrayList <BigEntityTableIndexItem>();
int trycount = 0;
while (true)
{
try
{
File.Delete(indexfile);
ProcessTraceUtil.Trace("删除旧文件完成");
break;
}
catch (System.IO.IOException ex)
{
Thread.Sleep(1);
trycount++;
if (trycount > 1000)
{
throw ex;
}
}
}
File.Move(newindexfile, indexfile);
ProcessTraceUtil.TraceMem("删除旧文件，重命名新文件完成", "m");
idxreader = null;
}
finally
{
tablelocker.ExitWriteLock();
}
string metafile = GetMetaFile(tablename);
mergeinfo.IndexMergePos = lasmargepos;
SerializerHelper.SerializerToXML(meta, metafile, true);
ProcessTraceUtil.Trace("更新元文件，更新索引完成");
}
catch (Exception ex)
{
Console.WriteLine("整理索引后面部分出错" + ex.ToString());
//LogManager.LogHelper.Instance.Error("整理索引后面部分出错", ex);
}
finally
{
if (idxreader != null)
{
idxreader.Dispose();
}
if (newwriter != null && !newwriter.Isdispose)
{
newwriter.Dispose();
}
if (File.Exists(newindexfile))
{
File.Delete(newindexfile);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
//LogManager.LogHelper.Instance.Error("整理索引出错", ex);
}
finally
{
//GC.Collect();
//ProcessTraceUtil.TraceMem("回收内存","m");
mergeinfo.IsMergin = false;
var info = ProcessTraceUtil.PrintTrace();
Console.WriteLine(info);
//LogManager.LogHelper.Instance.Debug("整理索引过程:" + info);
}
}
public LocalFileQueue(string queuename, string queuefilepath, bool canwrite = true, bool canread = true)
{
if (string.IsNullOrWhiteSpace(queuename))
{
throw new ArgumentNullException("queuename");
}
QueueName = queuename;
if (string.IsNullOrWhiteSpace(queuefilepath))
{
throw new ArgumentNullException("queuefilepath");
}
QueueFilePath = queuefilepath;
if (!File.Exists(queuefilepath))
{
FileInfo file = new FileInfo(queuefilepath);
if (!file.Directory.Exists)
{
try
{
file.Directory.Create();
}
catch (Exception ex)
{
throw new Exception("创建文件夹失败:" + file.Directory.FullName, ex);
}
}
}
if (canwrite)
{
//由于文件写是独占式的，当程序快速重启时，文件可能未及时释放
int trytimes = 0;
while (true)
{
try
{
//_queueWriter = ObjTextWriter.CreateWriter(queuefilepath, ObjTextReaderWriterEncodeType.jsonbuf);
_queueWriter = ObjTextWriter.CreateWriter(queuefilepath, ObjTextReaderWriterEncodeType.entitybuf);
_queueWriter.Flush();
break;
}
catch (Exception ex)
{
trytimes++;
if (trytimes >= 3)
{
throw ex;
}
Thread.Sleep(1000 * trytimes);
}
}
}
if (canread)
{
while (true)
{
if (File.Exists(queuefilepath))
{
break;
}
Thread.Sleep(1000);
}
_queueReader = ObjTextReader.CreateReader(queuefilepath);
FileInfo finfo = new FileInfo(queuefilepath);
QueueCfgFile = finfo.Directory.FullName + "\\" + queuename + ".cfg";
if (File.Exists(QueueCfgFile))
{
_logger = LJC.FrameWork.Comm.SerializerHelper.DeSerializerFile <LocalFileQueueCfg>(QueueCfgFile, true);
if (_logger.LastPos > 0)
{
_queueReader.SetPostion(_logger.LastPos);
}
}
else
{
_logger = new LocalFileQueueCfg();
_logger.LastChageTime = DateTime.Now;
_logger.QueueFile = queuefilepath;
SaveConfig();
}
}
_backtimer = new Timer(new TimerCallback(TimerAction), null, 0, 0);
}