/// <summary> /// /// </summary> /// <param name="tid"></param> /// <returns></returns> private static long ReadTagIndex(this DataFileSeriserbase datafile, int tid, out long offset, out short len) { len = datafile.ReadShort(0); var vsize = datafile.ReadInt(2); offset = vsize; lock (TagHeadOffsetManager.manager.LogHeadOffsets) { if (!TagHeadOffsetManager.manager.Contains(datafile.FileName)) { var data = datafile.Read(6, vsize); MarshalVarintCodeMemory vcm = new MarshalVarintCodeMemory(data.StartMemory, vsize); var ltmp = vcm.ToLongList(); int addroffset = (ltmp.Count - 1) / 2; List <int> ids = new List <int>(); List <long> addrs = new List <long>(); long lp = ltmp[1]; for (int i = 1; i < ltmp[0]; i++) { var lp2 = ltmp[i + 1] + lp; ids.Add((int)lp2); lp = lp2; } lp = ltmp[1 + addroffset]; for (int i = 1; i < ltmp[0]; i++) { var lp2 = ltmp[i + 1 + addroffset] + lp; addrs.Add(lp2); lp = lp2; } Dictionary <int, long> idaddrs = new Dictionary <int, long>(); for (int i = 0; i < ids.Count; i++) { idaddrs.Add(ids[i], addrs[i]); } TagHeadOffsetManager.manager.AddLogHead(datafile.FileName, idaddrs); if (idaddrs.ContainsKey(tid)) { return(idaddrs[tid]); } } else { var idaddrs = TagHeadOffsetManager.manager.Get(datafile.FileName); if (idaddrs.ContainsKey(tid)) { return(idaddrs[tid]); } } } return(-1); }
///// <summary> ///// ///// </summary> //static Dictionary<string, MarshalMemoryBlock> mMemoryCach = new Dictionary<string, MarshalMemoryBlock>(); /// <summary> /// /// </summary> /// <param name="datafile"></param> /// <param name="address"></param> /// <returns></returns> private static MarshalMemoryBlock GetDataMemory(DataFileSeriserbase datafile, long address) { //说明数据没有采用Zip压缩,可以直接读取使用 var dp = address; var datasize = datafile.ReadInt(dp); return(datafile.Read(dp + 4, datasize)); }
/// <summary> /// /// </summary> /// <param name="tid"></param> /// <returns></returns> private static AddressAndSize ReadTagIndex(this DataFileSeriserbase datafile, int tid, out long offset, out short len) { len = datafile.ReadShort(0); //读取时长 var vsize = datafile.ReadInt(10); offset = vsize + 6; if (vsize > 7) { lock (TagHeadOffsetManager.manager.LogHeadOffsets) { if (!TagHeadOffsetManager.manager.Contains(datafile.FileName)) { Dictionary <int, AddressAndSize> dtmps = new Dictionary <int, AddressAndSize>(); var idandaddress = datafile.Read(14, vsize); int id = idandaddress.ReadInt(0); long address = idandaddress.ReadLong(4); int size = 0; long ltmp; int nid = 0; for (int i = 1; i < vsize / 12; i++) { nid = idandaddress.ReadInt(i * 12); ltmp = idandaddress.ReadLong(i * 12 + 4); size = (int)(ltmp - address); dtmps.Add(id, new AddressAndSize() { Address = address, Size = size }); address = ltmp; id = nid; } dtmps.Add(id, new AddressAndSize() { Address = address, Size = (int)(datafile.Length - address) }); TagHeadOffsetManager.manager.AddLogHead(datafile.FileName, dtmps); if (dtmps.ContainsKey(tid)) { return(dtmps[tid]); } } else { var idaddrs = TagHeadOffsetManager.manager.GetLog(datafile.FileName); if (idaddrs.ContainsKey(tid)) { return(idaddrs[tid]); } } } } return(AddressAndSize.Empty); }
/// <summary> /// /// </summary> /// <param name="datafile"></param> /// <param name="tid"></param> /// <param name="offset"></param> /// <param name="dataTime"></param> /// <param name="timetick"></param> /// <returns></returns> private static MarshalMemoryBlock ReadTagDataBlock(DataFileSeriserbase datafile, int tid, long offset, DateTime dataTime, out int timetick, out int index) { int fileDuration, blockDuration = 0; int tagCount = 0; DateTime time; long blockpointer = 0; var dindex = ReadTagIndexInDataPointer(datafile, tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time); int blockcount = fileDuration * 60 / blockDuration; var startTime = datafile.ReadDateTime(16); var ttmp = (dataTime - startTime).TotalMinutes; int blockIndex = (int)(ttmp / blockDuration); if (ttmp % blockDuration > 0) { blockIndex++; } if (blockIndex > blockcount) { throw new Exception("DataPointer index is out of total block number"); } index = blockIndex; var headdata = datafile.Read(offset + blockpointer + dindex * blockcount * 12, blockcount * 12); var dataPointer = headdata.ReadInt(blockIndex * 12); //读取DataBlock的相对地址 var dataPointerbase = headdata.ReadLong(blockIndex * 12 + 4); //读取DataBlock的基地址 headdata.Dispose(); if (dataPointer >= 0) { return(GetDataMemory(datafile, dataPointerbase, dataPointer)); } else { var dp = dataPointerbase + (dataPointer & 0x7FFFFFFF); var datasize = datafile.ReadInt(dp); return(datafile.Read(dp + 4, datasize)); } }
/// <summary> /// /// </summary> /// <param name="datafile"></param> /// <param name="tid"></param> /// <param name="offset"></param> /// <param name="index"></param> /// <returns></returns> private static MarshalMemoryBlock ReadTagDataBlock(DataFileSeriserbase datafile, int tid, long offset, int index, out int timetick) { int fileDuration, blockDuration = 0; int tagCount = 0; DateTime time; long blockpointer = 0; var dindex = ReadTagIndexInDataPointer(datafile, tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time); int blockIndex = index; int blockcount = fileDuration * 60 / blockDuration; //var headdata = GetHeadBlock(datafile, offset + blockpointer, tagCount * blockcount * 12); //var dataPointer = headdata.ReadInt(dindex * 12 + blockIndex * tagCount * 12); //读取DataBlock的相对地址 //var dataPointerbase = headdata.ReadLong(dindex * 12 + blockIndex * tagCount * 12 + 4); //读取DataBlock的基地址 var headdata = datafile.Read(offset + blockpointer + dindex * blockcount * 12, blockcount * 12); var dataPointer = headdata.ReadInt(blockIndex * 12); //读取DataBlock的相对地址 var dataPointerbase = headdata.ReadLong(blockIndex * 12 + 4); //读取DataBlock的基地址 headdata.Dispose(); if (dataPointerbase > 0) { if (dataPointer >= 0) { return(GetDataMemory(datafile, dataPointerbase, dataPointer)); } else { var dp = dataPointerbase + (dataPointer & 0x7FFFFFFF); var datasize = datafile.ReadInt(dp); return(datafile.Read(dp + 4, datasize)); } } else { return(null); } }
///// <summary> ///// ///// </summary> //static Dictionary<string, MarshalMemoryBlock> mMemoryCach = new Dictionary<string, MarshalMemoryBlock>(); /// <summary> /// /// </summary> /// <param name="datafile"></param> /// <param name="address"></param> /// <returns></returns> private static MarshalMemoryBlock GetDataMemory(DataFileSeriserbase datafile, long address, int datapointer) { if (datapointer < 0) { //说明数据没有采用Zip压缩,可以直接读取使用 var dp = address + (datapointer & 0x7FFFFFFF); var datasize = datafile.ReadInt(dp); return(datafile.Read(dp + 4, datasize)); } else { return(DecodeMemoryCachManager.Manager.GetMemory(datafile, address, datapointer)); } }
/// <summary> /// /// </summary> /// <param name="datafile"></param> /// <param name="address"></param> private unsafe MarshalMemoryBlock ReadAndDecompressMemory(DataFileSeriserbase datafile, long address) { var vsize = datafile.ReadInt(address); var dsize = datafile.ReadInt(address + 4); var datas = datafile.Read(address + 8, vsize); int dtmp = 0; MarshalMemoryBlock mmb = new MarshalMemoryBlock(dsize, dsize); System.IO.Compression.BrotliDecoder.TryDecompress(new ReadOnlySpan <byte>((void *)datas.Handles[0], vsize), new Span <byte>((void *)mmb.Handles[0], dsize), out dtmp); if (dtmp != dsize) { LoggerService.Service.Warn("DataFileInfo", "解压缩数据长度不一致!" + datafile.FileName); } datas.Dispose(); return(mmb); }
/// <summary> /// 检测数据头部指针区域数据是否被缓存 /// </summary> /// <param name="datafile"></param> /// <param name="offset"></param> /// <param name="fileDuration"></param> /// <param name="blockDuration"></param> /// <param name="timetick"></param> /// <returns></returns> public static Dictionary <int, int> CheckBlockHeadCach(this DataFileSeriserbase datafile, long offset, out int tagCount, out int fileDuration, out int blockDuration, out int timetick, out long blockPointer, out DateTime time) { //文件头部结构:Pre DataRegion(8) + Next DataRegion(8) + Datatime(8)+tagcount(4)+ tagid sum(8) +file duration(4)+ block duration(4)+Time tick duration(4)+ { + tagid1+tagid2+...+tagidn }+ {[tag1 block point1(8) + tag2 block point1+ tag3 block point1+...] + [tag1 block point2(8) + tag2 block point2+ tag3 block point2+...]....} var dataoffset = offset + 16; //读取时间 time = datafile.ReadDateTime(dataoffset); dataoffset += 8; //读取变量个数 int count = datafile.ReadInt(dataoffset); dataoffset += 4; tagCount = count; //读取校验和 long idsum = datafile.ReadLong(dataoffset); dataoffset += 8; //读取单个文件的时长 fileDuration = datafile.ReadInt(dataoffset); dataoffset += 4; //读取数据块时长 blockDuration = datafile.ReadInt(dataoffset); dataoffset += 4; //读取时钟周期 timetick = datafile.ReadInt(dataoffset); dataoffset += 4; lock (TagHeadOffsetManager.manager) { if (!TagHeadOffsetManager.manager.Contains(idsum, count)) { //Tag id 列表经过压缩,内容格式为:DataSize + Data var dsize = datafile.ReadInt(dataoffset); if (dsize <= 0) { tagCount = 0; fileDuration = 0; blockDuration = 0; timetick = 0; blockPointer = 0; return(new Dictionary <int, int>()); } dataoffset += 4; blockPointer = dataoffset + dsize - offset; var dtmp = new Dictionary <int, int>(); using (var dd = datafile.Read(dataoffset, dsize)) { MarshalVarintCodeMemory vcm = new MarshalVarintCodeMemory(dd.StartMemory, dsize); var ltmp = vcm.ToIntList(); //vcm.Dispose(); if (ltmp.Count > 0) { int preid = ltmp[0]; dtmp.Add(preid, 0); for (int i = 1; i < ltmp.Count; i++) { var id = ltmp[i] + preid; dtmp.Add(id, i); preid = id; } } TagHeadOffsetManager.manager.Add(idsum, count, dtmp, blockPointer); dd.Dispose(); } return(dtmp); } else { var re = TagHeadOffsetManager.manager.Get(idsum, count); blockPointer = re.Item2; return(re.Item1); } } }
/// <summary> /// /// </summary> /// <param name="datafile"></param> /// <param name="startTime"></param> /// <param name="tid"></param> /// <param name="offset"></param> /// <param name="start"></param> /// <param name="end"></param> /// <param name="timetick"></param> /// <returns></returns> private static IEnumerable <Tuple <MarshalMemoryBlock, DateTime, DateTime, int> > ReadTagDataBlock2(DataFileSeriserbase datafile, int tid, long offset, DateTime start, DateTime end) { int fileDuration, blockDuration = 0; int tagCount = 0; long blockpointer = 0; int timetick = 0; DateTime time; //var tagIndex = ReadTagIndexInDataPointer(datafile,tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time); ReadRegionHead(datafile, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time); var tagIndex = tid % tagCount; int blockcount = fileDuration * 60 / blockDuration; //读取文件开始时间 var startTime = datafile.ReadDateTime(0); DateTime sstart = start; DateTime send = end; int buffersize = 1024 * 1024 * 2; //分配读缓存 IntPtr mdataBuffer = Marshal.AllocHGlobal(buffersize); long mbufferadderss = 0; int bufferLen = 0; //var headdata = GetHeadBlock(datafile, offset + blockpointer, tagCount * blockcount * 12); var headdata = datafile.Read(offset + blockpointer + tagIndex * blockcount * 8, blockcount * 8); while (sstart < end) { var ttmp = Math.Round((sstart - startTime).TotalSeconds, 3); var vv = blockDuration * 60 - (ttmp % (blockDuration * 60)); send = sstart.AddSeconds(vv); if (send > end) { send = end; } int blockindex = (int)(ttmp / (blockDuration * 60)); if (blockindex >= blockcount) { break; //throw new Exception("DataPointer index is out of total block number"); } var dataPointer = headdata.ReadInt(blockindex * 8); //读取DataBlock的相对地址 if (dataPointer > 0) { MarshalMemoryBlock vmm = null; //说明数据没有采用Zip压缩,可以直接读取使用 var dp = dataPointer; int datasize = 0; int dataloc = 0; if (dp >= mbufferadderss && (dp - mbufferadderss + 4) <= bufferLen && (dp - mbufferadderss + 4 + MemoryHelper.ReadInt32(mdataBuffer, dp - mbufferadderss)) <= bufferLen) { datasize = MemoryHelper.ReadInt32(mdataBuffer, dp - mbufferadderss); dataloc = (int)(dp - mbufferadderss + 4); } else { bufferLen = datafile.Read(mdataBuffer, dp, buffersize); mbufferadderss = dp; datasize = MemoryHelper.ReadInt32(mdataBuffer, 0); dataloc = (int)(dp - mbufferadderss + 4); } if (datasize > 0 && datasize < datafile.Length) { vmm = new MarshalMemoryBlock(datasize, datasize); MemoryHelper.MemoryCopy(mdataBuffer, dataloc, vmm.Buffers[0], 0, datasize); } if (vmm != null) { yield return(new Tuple <MarshalMemoryBlock, DateTime, DateTime, int>(vmm, sstart, send, timetick)); } } sstart = send; } headdata.Dispose(); Marshal.FreeHGlobal(mdataBuffer); }
/// <summary> /// /// </summary> /// <param name="datafile"></param> /// <param name="tid"></param> /// <param name="offset"></param> /// <param name="dataTimes"></param> /// <param name="timetick"></param> /// <returns></returns> private static Dictionary <MarshalMemoryBlock, Tuple <List <DateTime>, int> > ReadTagDataBlock2(DataFileSeriserbase datafile, int tid, long offset, List <DateTime> dataTimes, out int timetick) { int fileDuration, blockDuration = 0; int tagCount = 0; long blockpointer = 0; DateTime time; //var tagIndex = ReadTagIndexInDataPointer(datafile,tid, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time); ReadRegionHead(datafile, offset, out tagCount, out fileDuration, out blockDuration, out timetick, out blockpointer, out time); var tagIndex = tid % tagCount; Dictionary <long, MarshalMemoryBlock> rtmp = new Dictionary <long, MarshalMemoryBlock>(); Dictionary <MarshalMemoryBlock, Tuple <List <DateTime>, int> > re = new Dictionary <MarshalMemoryBlock, Tuple <List <DateTime>, int> >(); if (tagCount == 0) { return(re); } int blockcount = fileDuration * 60 / blockDuration; var startTime = datafile.ReadDateTime(0); int buffersize = 1024 * 1024 * 2; //分配读缓存 IntPtr mdataBuffer = Marshal.AllocHGlobal(buffersize); long mbufferadderss = 0; int bufferLen = buffersize; // var headdata = GetHeadBlock(datafile, offset + blockpointer, tagCount * blockcount * 12); var headdata = datafile.Read(offset + blockpointer + tagIndex * blockcount * 8, blockcount * 8); long mLastBuffer = 0; int mLastDataLoc = 0; int mLastDataSize = 0; MarshalMemoryBlock vmm = null; foreach (var vdd in dataTimes) { var ttmp = (vdd - startTime).TotalMinutes; int blockindex = (int)(ttmp / blockDuration); if (blockindex > blockcount) { throw new Exception("DataPointer index is out of total block number"); } //var dataPointer = headdata.ReadInt(tagIndex * blockcount * 12 + blockindex * 12); //读取DataBlock的相对地址 //var dataPointerbase = headdata.ReadLong(tagIndex * blockcount * 12 + blockindex * 12 + 4); //读取DataBlock的基地址 var dataPointer = headdata.ReadInt(blockindex * 8); //读取DataBlock的相对地址 //var dataPointerbase = headdata.ReadLong(blockindex * 12 + 4); //读取DataBlock的基地址 if (dataPointer > 0) { ////var datasize = datafile.ReadInt(dataPointer); //读取DataBlock 的大小 //var vmm = GetDataMemory(datafile, dataPointerbase, dataPointer); //说明数据没有采用Zip压缩,可以直接读取使用 var dp = dataPointer; int datasize = 0; int dataloc = 0; if (dp >= mbufferadderss && (dp - mbufferadderss + 4) <= bufferLen && (dp - mbufferadderss + 4 + MemoryHelper.ReadInt32(mdataBuffer, dp - mbufferadderss)) <= bufferLen) { datasize = MemoryHelper.ReadInt32(mdataBuffer, dp - mbufferadderss); dataloc = (int)(dp - mbufferadderss + 4); } else { bufferLen = datafile.Read(mdataBuffer, dp, buffersize); mbufferadderss = dp; datasize = MemoryHelper.ReadInt32(mdataBuffer, 0); dataloc = (int)(dp - mbufferadderss + 4); } if (datasize > 0 && (mLastBuffer != mbufferadderss || mLastDataLoc != dataloc || mLastDataSize != datasize)) { vmm = new MarshalMemoryBlock(datasize, datasize); MemoryHelper.MemoryCopy(mdataBuffer, dataloc, vmm.Buffers[0], 0, datasize); mLastBuffer = mbufferadderss; mLastDataLoc = dataloc; mLastDataSize = datasize; } else if (datasize <= 0) { vmm = null; } if (vmm != null) { if (!rtmp.ContainsKey(dataPointer)) { //var rmm = datafile.Read(dataPointer + 4, datasize); if (!re.ContainsKey(vmm)) { re.Add(vmm, new Tuple <List <DateTime>, int>(new List <DateTime>() { vdd }, blockindex)); } else { re[vmm].Item1.Add(vdd); } rtmp.Add(dataPointer, vmm); } else { //var rmm = rtmp[dataPointer]; if (!re.ContainsKey(vmm)) { re.Add(vmm, new Tuple <List <DateTime>, int>(new List <DateTime>() { vdd }, blockindex)); } else { re[vmm].Item1.Add(vdd); } } } } } headdata.Dispose(); Marshal.FreeHGlobal(mdataBuffer); return(re); }
/// <summary> /// /// </summary> /// <param name="datafile"></param> /// <param name="address"></param> /// <returns></returns> public MarshalMemoryBlock GetMemory(DataFileSeriserbase datafile, long address, int len) { Stopwatch sw = new Stopwatch(); sw.Start(); try { lock (mCacheDatas) { string skey = System.IO.Path.GetFileNameWithoutExtension(datafile.FileName) + address; if (mCacheDatas.ContainsKey(skey) && mCacheDatas[skey] != null && mCacheDatas[skey].DataBlock != null) { mCacheDatas[skey].LastAccessTime = DateTime.Now; return(mCacheDatas[skey].DataBlock); } else { DateTime dnow = DateTime.Now; var mh = datafile.Read(address, len); if (!mCacheDatas.ContainsKey(skey)) { mCacheDatas.Add(skey, new HeadDataPointCachItem() { DataBlock = mh, LastAccessTime = dnow, Name = skey }); } else { mCacheDatas[skey].DataBlock = mh; } if (mCacheDatas.Count > 24) { Task.Run(() => { try { lock (mCacheDatas) { foreach (var vv in mCacheDatas.Values.OrderBy(e => e.LastAccessTime)) { if ((dnow - vv.LastAccessTime).TotalDays >= 1) { mCacheDatas.Remove(vv.Name); } vv.DataBlock.Dispose(); } } } catch { } }); } return(mh); } } } finally { sw.Stop(); Debug.Print("获取头部指针:" + sw.ElapsedMilliseconds); } }