/// <summary>
///
/// </summary>
/// <param name="tid"></param>
/// <returns></returns>
private static long ReadTagIndex(this DataFileSeriserbase datafile, int tid, out long offset, out short len)
{
len = datafile.ReadShort(0);
var vsize = datafile.ReadInt(2);
offset = vsize;
lock (TagHeadOffsetManager.manager.LogHeadOffsets)
{
if (!TagHeadOffsetManager.manager.Contains(datafile.FileName))
{
var data = datafile.Read(6, vsize);
MarshalVarintCodeMemory vcm = new MarshalVarintCodeMemory(data.StartMemory, vsize);
var ltmp = vcm.ToLongList();
int addroffset = (ltmp.Count - 1) / 2;
List <int> ids = new List <int>();
List <long> addrs = new List <long>();
long lp = ltmp[1];
for (int i = 1; i < ltmp[0]; i++)
{
var lp2 = ltmp[i + 1] + lp;
ids.Add((int)lp2);
lp = lp2;
}
lp = ltmp[1 + addroffset];
for (int i = 1; i < ltmp[0]; i++)
{
var lp2 = ltmp[i + 1 + addroffset] + lp;
addrs.Add(lp2);
lp = lp2;
}
Dictionary <int, long> idaddrs = new Dictionary <int, long>();
for (int i = 0; i < ids.Count; i++)
{
idaddrs.Add(ids[i], addrs[i]);
}
TagHeadOffsetManager.manager.AddLogHead(datafile.FileName, idaddrs);
if (idaddrs.ContainsKey(tid))
{
return(idaddrs[tid]);
}
}
else
{
var idaddrs = TagHeadOffsetManager.manager.Get(datafile.FileName);
if (idaddrs.ContainsKey(tid))
{
return(idaddrs[tid]);
}
}
}
return(-1);
}
///// <summary>
/////
///// </summary>
//static Dictionary<string, MarshalMemoryBlock> mMemoryCach = new Dictionary<string, MarshalMemoryBlock>();
/// <summary>
///
/// </summary>
/// <param name="datafile"></param>
/// <param name="address"></param>
/// <returns></returns>
private static MarshalMemoryBlock GetDataMemory(DataFileSeriserbase datafile, long address)
{
//说明数据没有采用Zip压缩，可以直接读取使用
var dp = address;
var datasize = datafile.ReadInt(dp);
return(datafile.Read(dp + 4, datasize));
}
/// <summary>
///
/// </summary>
/// <param name="tid"></param>
/// <returns></returns>
private static AddressAndSize ReadTagIndex(this DataFileSeriserbase datafile, int tid, out long offset, out short len)
{
len = datafile.ReadShort(0); //读取时长
var vsize = datafile.ReadInt(10);
offset = vsize + 6;
if (vsize > 7)
{
lock (TagHeadOffsetManager.manager.LogHeadOffsets)
{
if (!TagHeadOffsetManager.manager.Contains(datafile.FileName))
{
Dictionary <int, AddressAndSize> dtmps = new Dictionary <int, AddressAndSize>();
var idandaddress = datafile.Read(14, vsize);
int id = idandaddress.ReadInt(0);
long address = idandaddress.ReadLong(4);
int size = 0;
long ltmp;
int nid = 0;
for (int i = 1; i < vsize / 12; i++)
{
nid = idandaddress.ReadInt(i * 12);
ltmp = idandaddress.ReadLong(i * 12 + 4);
size = (int)(ltmp - address);
dtmps.Add(id, new AddressAndSize()
{
Address = address, Size = size
});
address = ltmp;
id = nid;
}
dtmps.Add(id, new AddressAndSize()
{
Address = address, Size = (int)(datafile.Length - address)
});
TagHeadOffsetManager.manager.AddLogHead(datafile.FileName, dtmps);
if (dtmps.ContainsKey(tid))
{
return(dtmps[tid]);
}
}
else
{
var idaddrs = TagHeadOffsetManager.manager.GetLog(datafile.FileName);
if (idaddrs.ContainsKey(tid))
{
return(idaddrs[tid]);
}
}
}
}
return(AddressAndSize.Empty);
}
/// <summary>
///
/// </summary>
/// <param name="datafile"></param>
/// <param name="tid"></param>
/// <param name="offset"></param>
/// <param name="dataTime"></param>
/// <param name="timetick"></param>
/// <returns></returns>
private static MarshalMemoryBlock ReadTagDataBlock(DataFileSeriserbase datafile, int tid, long offset, DateTime dataTime, out int timetick, out int index)
{
int fileDuration, blockDuration = 0;
int tagCount = 0;
DateTime time;
long blockpointer = 0;
var dindex = ReadTagIndexInDataPointer(datafile, tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time);
int blockcount = fileDuration * 60 / blockDuration;
var startTime = datafile.ReadDateTime(16);
var ttmp = (dataTime - startTime).TotalMinutes;
int blockIndex = (int)(ttmp / blockDuration);
if (ttmp % blockDuration > 0)
{
blockIndex++;
}
if (blockIndex > blockcount)
{
throw new Exception("DataPointer index is out of total block number");
}
index = blockIndex;
var headdata = datafile.Read(offset + blockpointer + dindex * blockcount * 12, blockcount * 12);
var dataPointer = headdata.ReadInt(blockIndex * 12); //读取DataBlock的相对地址
var dataPointerbase = headdata.ReadLong(blockIndex * 12 + 4); //读取DataBlock的基地址
headdata.Dispose();
if (dataPointer >= 0)
{
return(GetDataMemory(datafile, dataPointerbase, dataPointer));
}
else
{
var dp = dataPointerbase + (dataPointer & 0x7FFFFFFF);
var datasize = datafile.ReadInt(dp);
return(datafile.Read(dp + 4, datasize));
}
}
/// <summary>
///
/// </summary>
/// <param name="datafile"></param>
/// <param name="tid"></param>
/// <param name="offset"></param>
/// <param name="index"></param>
/// <returns></returns>
private static MarshalMemoryBlock ReadTagDataBlock(DataFileSeriserbase datafile, int tid, long offset, int index, out int timetick)
{
int fileDuration, blockDuration = 0;
int tagCount = 0;
DateTime time;
long blockpointer = 0;
var dindex = ReadTagIndexInDataPointer(datafile, tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time);
int blockIndex = index;
int blockcount = fileDuration * 60 / blockDuration;
//var headdata = GetHeadBlock(datafile, offset + blockpointer, tagCount * blockcount * 12);
//var dataPointer = headdata.ReadInt(dindex * 12 + blockIndex * tagCount * 12); //读取DataBlock的相对地址
//var dataPointerbase = headdata.ReadLong(dindex * 12 + blockIndex * tagCount * 12 + 4); //读取DataBlock的基地址
var headdata = datafile.Read(offset + blockpointer + dindex * blockcount * 12, blockcount * 12);
var dataPointer = headdata.ReadInt(blockIndex * 12); //读取DataBlock的相对地址
var dataPointerbase = headdata.ReadLong(blockIndex * 12 + 4); //读取DataBlock的基地址
headdata.Dispose();
if (dataPointerbase > 0)
{
if (dataPointer >= 0)
{
return(GetDataMemory(datafile, dataPointerbase, dataPointer));
}
else
{
var dp = dataPointerbase + (dataPointer & 0x7FFFFFFF);
var datasize = datafile.ReadInt(dp);
return(datafile.Read(dp + 4, datasize));
}
}
else
{
return(null);
}
}
///// <summary>
/////
///// </summary>
//static Dictionary<string, MarshalMemoryBlock> mMemoryCach = new Dictionary<string, MarshalMemoryBlock>();
/// <summary>
///
/// </summary>
/// <param name="datafile"></param>
/// <param name="address"></param>
/// <returns></returns>
private static MarshalMemoryBlock GetDataMemory(DataFileSeriserbase datafile, long address, int datapointer)
{
if (datapointer < 0)
{
//说明数据没有采用Zip压缩，可以直接读取使用
var dp = address + (datapointer & 0x7FFFFFFF);
var datasize = datafile.ReadInt(dp);
return(datafile.Read(dp + 4, datasize));
}
else
{
return(DecodeMemoryCachManager.Manager.GetMemory(datafile, address, datapointer));
}
}
/// <summary>
///
/// </summary>
/// <param name="datafile"></param>
/// <param name="address"></param>
private unsafe MarshalMemoryBlock ReadAndDecompressMemory(DataFileSeriserbase datafile, long address)
{
var vsize = datafile.ReadInt(address);
var dsize = datafile.ReadInt(address + 4);
var datas = datafile.Read(address + 8, vsize);
int dtmp = 0;
MarshalMemoryBlock mmb = new MarshalMemoryBlock(dsize, dsize);
System.IO.Compression.BrotliDecoder.TryDecompress(new ReadOnlySpan <byte>((void *)datas.Handles[0], vsize), new Span <byte>((void *)mmb.Handles[0], dsize), out dtmp);
if (dtmp != dsize)
{
LoggerService.Service.Warn("DataFileInfo", "解压缩数据长度不一致!" + datafile.FileName);
}
datas.Dispose();
return(mmb);
}
/// <summary>
/// 检测数据头部指针区域数据是否被缓存
/// </summary>
/// <param name="datafile"></param>
/// <param name="offset"></param>
/// <param name="fileDuration"></param>
/// <param name="blockDuration"></param>
/// <param name="timetick"></param>
/// <returns></returns>
public static Dictionary <int, int> CheckBlockHeadCach(this DataFileSeriserbase datafile, long offset, out int tagCount, out int fileDuration, out int blockDuration, out int timetick, out long blockPointer, out DateTime time)
{
//文件头部结构:Pre DataRegion(8) + Next DataRegion(8) + Datatime(8)+tagcount(4)+ tagid sum(8) +file duration(4)+ block duration(4)+Time tick duration(4)+ { + tagid1+tagid2+...+tagidn }+ {[tag1 block point1(8) + tag2 block point1+ tag3 block point1+...] + [tag1 block point2(8) + tag2 block point2+ tag3 block point2+...]....}
var dataoffset = offset + 16;
//读取时间
time = datafile.ReadDateTime(dataoffset);
dataoffset += 8;
//读取变量个数
int count = datafile.ReadInt(dataoffset);
dataoffset += 4;
tagCount = count;
//读取校验和
long idsum = datafile.ReadLong(dataoffset);
dataoffset += 8;
//读取单个文件的时长
fileDuration = datafile.ReadInt(dataoffset);
dataoffset += 4;
//读取数据块时长
blockDuration = datafile.ReadInt(dataoffset);
dataoffset += 4;
//读取时钟周期
timetick = datafile.ReadInt(dataoffset);
dataoffset += 4;
lock (TagHeadOffsetManager.manager)
{
if (!TagHeadOffsetManager.manager.Contains(idsum, count))
{
//Tag id 列表经过压缩，内容格式为:DataSize + Data
var dsize = datafile.ReadInt(dataoffset);
if (dsize <= 0)
{
tagCount = 0;
fileDuration = 0;
blockDuration = 0;
timetick = 0;
blockPointer = 0;
return(new Dictionary <int, int>());
}
dataoffset += 4;
blockPointer = dataoffset + dsize - offset;
var dtmp = new Dictionary <int, int>();
using (var dd = datafile.Read(dataoffset, dsize))
{
MarshalVarintCodeMemory vcm = new MarshalVarintCodeMemory(dd.StartMemory, dsize);
var ltmp = vcm.ToIntList();
//vcm.Dispose();
if (ltmp.Count > 0)
{
int preid = ltmp[0];
dtmp.Add(preid, 0);
for (int i = 1; i < ltmp.Count; i++)
{
var id = ltmp[i] + preid;
dtmp.Add(id, i);
preid = id;
}
}
TagHeadOffsetManager.manager.Add(idsum, count, dtmp, blockPointer);
dd.Dispose();
}
return(dtmp);
}
else
{
var re = TagHeadOffsetManager.manager.Get(idsum, count);
blockPointer = re.Item2;
return(re.Item1);
}
}
}
/// <summary>
///
/// </summary>
/// <param name="datafile"></param>
/// <param name="startTime"></param>
/// <param name="tid"></param>
/// <param name="offset"></param>
/// <param name="start"></param>
/// <param name="end"></param>
/// <param name="timetick"></param>
/// <returns></returns>
private static IEnumerable <Tuple <MarshalMemoryBlock, DateTime, DateTime, int> > ReadTagDataBlock2(DataFileSeriserbase datafile, int tid, long offset, DateTime start, DateTime end)
{
int fileDuration, blockDuration = 0;
int tagCount = 0;
long blockpointer = 0;
int timetick = 0;
DateTime time;
//var tagIndex = ReadTagIndexInDataPointer(datafile,tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time);
ReadRegionHead(datafile, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time);
var tagIndex = tid % tagCount;
int blockcount = fileDuration * 60 / blockDuration;
//读取文件开始时间
var startTime = datafile.ReadDateTime(0);
DateTime sstart = start;
DateTime send = end;
int buffersize = 1024 * 1024 * 2;
//分配读缓存
IntPtr mdataBuffer = Marshal.AllocHGlobal(buffersize);
long mbufferadderss = 0;
int bufferLen = 0;
//var headdata = GetHeadBlock(datafile, offset + blockpointer, tagCount * blockcount * 12);
var headdata = datafile.Read(offset + blockpointer + tagIndex * blockcount * 8, blockcount * 8);
while (sstart < end)
{
var ttmp = Math.Round((sstart - startTime).TotalSeconds, 3);
var vv = blockDuration * 60 - (ttmp % (blockDuration * 60));
send = sstart.AddSeconds(vv);
if (send > end)
{
send = end;
}
int blockindex = (int)(ttmp / (blockDuration * 60));
if (blockindex >= blockcount)
{
break;
//throw new Exception("DataPointer index is out of total block number");
}
var dataPointer = headdata.ReadInt(blockindex * 8); //读取DataBlock的相对地址
if (dataPointer > 0)
{
MarshalMemoryBlock vmm = null;
//说明数据没有采用Zip压缩，可以直接读取使用
var dp = dataPointer;
int datasize = 0;
int dataloc = 0;
if (dp >= mbufferadderss && (dp - mbufferadderss + 4) <= bufferLen && (dp - mbufferadderss + 4 + MemoryHelper.ReadInt32(mdataBuffer, dp - mbufferadderss)) <= bufferLen)
{
datasize = MemoryHelper.ReadInt32(mdataBuffer, dp - mbufferadderss);
dataloc = (int)(dp - mbufferadderss + 4);
}
else
{
bufferLen = datafile.Read(mdataBuffer, dp, buffersize);
mbufferadderss = dp;
datasize = MemoryHelper.ReadInt32(mdataBuffer, 0);
dataloc = (int)(dp - mbufferadderss + 4);
}
if (datasize > 0 && datasize < datafile.Length)
{
vmm = new MarshalMemoryBlock(datasize, datasize);
MemoryHelper.MemoryCopy(mdataBuffer, dataloc, vmm.Buffers[0], 0, datasize);
}
if (vmm != null)
{
yield return(new Tuple <MarshalMemoryBlock, DateTime, DateTime, int>(vmm, sstart, send, timetick));
}
}
sstart = send;
}
headdata.Dispose();
Marshal.FreeHGlobal(mdataBuffer);
}
/// <summary>
///
/// </summary>
/// <param name="datafile"></param>
/// <param name="tid"></param>
/// <param name="offset"></param>
/// <param name="dataTimes"></param>
/// <param name="timetick"></param>
/// <returns></returns>
private static Dictionary <MarshalMemoryBlock, Tuple <List <DateTime>, int> > ReadTagDataBlock2(DataFileSeriserbase datafile, int tid, long offset, List <DateTime> dataTimes, out int timetick)
{
int fileDuration, blockDuration = 0;
int tagCount = 0;
long blockpointer = 0;
DateTime time;
//var tagIndex = ReadTagIndexInDataPointer(datafile,tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time);
ReadRegionHead(datafile, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time);
var tagIndex = tid % tagCount;
Dictionary <long, MarshalMemoryBlock> rtmp = new Dictionary <long, MarshalMemoryBlock>();
Dictionary <MarshalMemoryBlock, Tuple <List <DateTime>, int> > re = new Dictionary <MarshalMemoryBlock, Tuple <List <DateTime>, int> >();
if (tagCount == 0)
{
return(re);
}
int blockcount = fileDuration * 60 / blockDuration;
var startTime = datafile.ReadDateTime(0);
int buffersize = 1024 * 1024 * 2;
//分配读缓存
IntPtr mdataBuffer = Marshal.AllocHGlobal(buffersize);
long mbufferadderss = 0;
int bufferLen = buffersize;
// var headdata = GetHeadBlock(datafile, offset + blockpointer, tagCount * blockcount * 12);
var headdata = datafile.Read(offset + blockpointer + tagIndex * blockcount * 8, blockcount * 8);
long mLastBuffer = 0;
int mLastDataLoc = 0;
int mLastDataSize = 0;
MarshalMemoryBlock vmm = null;
foreach (var vdd in dataTimes)
{
var ttmp = (vdd - startTime).TotalMinutes;
int blockindex = (int)(ttmp / blockDuration);
if (blockindex > blockcount)
{
throw new Exception("DataPointer index is out of total block number");
}
//var dataPointer = headdata.ReadInt(tagIndex * blockcount * 12 + blockindex * 12); //读取DataBlock的相对地址
//var dataPointerbase = headdata.ReadLong(tagIndex * blockcount * 12 + blockindex * 12 + 4); //读取DataBlock的基地址
var dataPointer = headdata.ReadInt(blockindex * 8); //读取DataBlock的相对地址
//var dataPointerbase = headdata.ReadLong(blockindex * 12 + 4); //读取DataBlock的基地址
if (dataPointer > 0)
{
////var datasize = datafile.ReadInt(dataPointer); //读取DataBlock 的大小
//var vmm = GetDataMemory(datafile, dataPointerbase, dataPointer);
//说明数据没有采用Zip压缩，可以直接读取使用
var dp = dataPointer;
int datasize = 0;
int dataloc = 0;
if (dp >= mbufferadderss && (dp - mbufferadderss + 4) <= bufferLen && (dp - mbufferadderss + 4 + MemoryHelper.ReadInt32(mdataBuffer, dp - mbufferadderss)) <= bufferLen)
{
datasize = MemoryHelper.ReadInt32(mdataBuffer, dp - mbufferadderss);
dataloc = (int)(dp - mbufferadderss + 4);
}
else
{
bufferLen = datafile.Read(mdataBuffer, dp, buffersize);
mbufferadderss = dp;
datasize = MemoryHelper.ReadInt32(mdataBuffer, 0);
dataloc = (int)(dp - mbufferadderss + 4);
}
if (datasize > 0 && (mLastBuffer != mbufferadderss || mLastDataLoc != dataloc || mLastDataSize != datasize))
{
vmm = new MarshalMemoryBlock(datasize, datasize);
MemoryHelper.MemoryCopy(mdataBuffer, dataloc, vmm.Buffers[0], 0, datasize);
mLastBuffer = mbufferadderss;
mLastDataLoc = dataloc;
mLastDataSize = datasize;
}
else if (datasize <= 0)
{
vmm = null;
}
if (vmm != null)
{
if (!rtmp.ContainsKey(dataPointer))
{
//var rmm = datafile.Read(dataPointer + 4, datasize);
if (!re.ContainsKey(vmm))
{
re.Add(vmm, new Tuple <List <DateTime>, int>(new List <DateTime>()
{
vdd
}, blockindex));
}
else
{
re[vmm].Item1.Add(vdd);
}
rtmp.Add(dataPointer, vmm);
}
else
{
//var rmm = rtmp[dataPointer];
if (!re.ContainsKey(vmm))
{
re.Add(vmm, new Tuple <List <DateTime>, int>(new List <DateTime>()
{
vdd
}, blockindex));
}
else
{
re[vmm].Item1.Add(vdd);
}
}
}
}
}
headdata.Dispose();
Marshal.FreeHGlobal(mdataBuffer);
return(re);
}
/// <summary>
///
/// </summary>
/// <param name="datafile"></param>
/// <param name="address"></param>
/// <returns></returns>
public MarshalMemoryBlock GetMemory(DataFileSeriserbase datafile, long address, int len)
{
Stopwatch sw = new Stopwatch();
sw.Start();
try
{
lock (mCacheDatas)
{
string skey = System.IO.Path.GetFileNameWithoutExtension(datafile.FileName) + address;
if (mCacheDatas.ContainsKey(skey) && mCacheDatas[skey] != null && mCacheDatas[skey].DataBlock != null)
{
mCacheDatas[skey].LastAccessTime = DateTime.Now;
return(mCacheDatas[skey].DataBlock);
}
else
{
DateTime dnow = DateTime.Now;
var mh = datafile.Read(address, len);
if (!mCacheDatas.ContainsKey(skey))
{
mCacheDatas.Add(skey, new HeadDataPointCachItem()
{
DataBlock = mh, LastAccessTime = dnow, Name = skey
});
}
else
{
mCacheDatas[skey].DataBlock = mh;
}
if (mCacheDatas.Count > 24)
{
Task.Run(() =>
{
try
{
lock (mCacheDatas)
{
foreach (var vv in mCacheDatas.Values.OrderBy(e => e.LastAccessTime))
{
if ((dnow - vv.LastAccessTime).TotalDays >= 1)
{
mCacheDatas.Remove(vv.Name);
}
vv.DataBlock.Dispose();
}
}
}
catch
{
}
});
}
return(mh);
}
}
}
finally
{
sw.Stop();
Debug.Print("获取头部指针:" + sw.ElapsedMilliseconds);
}
}
